From 263e5268c8cb81ecc83fe792413b3dabdbd9938d Mon Sep 17 00:00:00 2001 From: rsasaki0109 Date: Tue, 28 Jul 2026 02:07:31 +0900 Subject: [PATCH 1/2] Add SpatialRust 1.2 streaming release gate --- .github/workflows/ci.yml | 32 ++ CHANGELOG.md | 34 +- Cargo.toml | 62 +-- README.md | 6 + crates/spatialrust-platform/src/lib.rs | 2 + crates/spatialrust-platform/src/stability.rs | 47 +++ crates/spatialrust-platform/src/streaming.rs | 375 ++++++++++++++++++ crates/spatialrust-py/Cargo.toml | 2 +- crates/spatialrust-records/src/bounded.rs | 10 +- crates/spatialrust/Cargo.toml | 5 + .../examples/streaming_1_2_release_gate.rs | 105 +++++ docs/API_STABILITY.md | 14 +- docs/ARCHITECTURE.md | 11 + docs/ROADMAP.md | 12 +- docs/STREAMING_MIGRATION.md | 85 ++++ docs/STREAMING_RELEASE_RECEIPT.md | 44 ++ ...6-07-27_epic-126-streaming-release-gate.md | 34 ++ 17 files changed, 840 insertions(+), 40 deletions(-) create mode 100644 crates/spatialrust-platform/src/streaming.rs create mode 100644 crates/spatialrust/examples/streaming_1_2_release_gate.rs create mode 100644 docs/STREAMING_MIGRATION.md create mode 100644 docs/STREAMING_RELEASE_RECEIPT.md create mode 100644 docs/receipts/2026-07-27_epic-126-streaming-release-gate.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5644245..37e2471 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,18 @@ jobs: args: --workspace - name: spatialrust-serde args: -p spatialrust --features serde + - name: spatialrust-records-receipt-json + args: -p spatialrust-records --features receipt-json + - name: spatialrust-records-receipt-json-meta + args: -p spatialrust --features records-receipt-json + - name: spatialrust-io-streaming-all + args: -p spatialrust-io --no-default-features --features io-pcd,io-ply,io-las,io-laz,io-copc-http,streaming + - name: spatialrust-pipeline-streaming + args: -p spatialrust-pipeline --no-default-features --features pipeline-streaming + - name: spatialrust-pipeline-streaming-meta + args: -p spatialrust --no-default-features --features pipeline-streaming + - name: spatialrust-streaming-cli + args: -p spatialrust --no-default-features --features streaming-cli - name: spatialrust-io-pcd args: -p spatialrust-io --features io-pcd - name: spatialrust-io-pcd-meta @@ -290,6 +302,26 @@ jobs: cargo run -p spatialrust --no-default-features --features platform --example vision_2_release_gate cargo run -p spatialrust --no-default-features --features image-io-standard,vision-video --example video_tracking_e2e + streaming-platform-conformance: + name: Streaming 1.2 conformance (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Test bounded-streaming contracts and release gate + run: | + cargo test -p spatialrust-records --features receipt-json + cargo test -p spatialrust-io --no-default-features --features io-pcd,io-ply,io-las,io-laz,io-copc-http,streaming + cargo test -p spatialrust-pipeline --no-default-features --features pipeline-streaming + cargo test -p spatialrust --no-default-features --features streaming-cli --test streaming_cli + cargo test -p spatialrust-platform streaming + cargo run -p spatialrust --no-default-features --features platform,pipeline-streaming --example streaming_1_2_release_gate + bench: name: Benchmark compile runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index b6be22c..e11b464 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ removed no sooner than the next major (see `docs/API_STABILITY.md`). ## [Unreleased] +## [1.2.0] — 2026-07-27 + ### Added - **SpatialRust 1.2 bounded-streaming contract (Epic 121)**: @@ -38,8 +40,37 @@ removed no sooner than the next major (see `docs/API_STABILITY.md`). ownership-preserving `PointCloud::into_parts` inverse and mutable column iteration needed to clear recycled storage without allocating field names. +- **Bounded point-cloud IO and spool contracts (Epic 123)**: feature-gated + PCD, PLY, LAS/LAZ, and deterministic local/HTTP COPC sources emit leased + chunks without whole-cloud materialization. Exact-count and open-ended sinks, + source-driven COPC output, bounded ASCII records, and extent-limited + temporary storage fail before exceeding their declared limits. + +- **Chunk-safe streaming operations (Epic 124)**: shared-budget crop and affine + transform adapters, compensated global position reductions, and + deterministic external voxel centroids compose outside + `spatialrust-core`. Voxel output is invariant to source chunk and sort-run + size, with explicit spill/run/file-handle limits. + +- **Rust, CLI, and Python streaming workflows (Epic 125)**: + `StreamingPipeline` provides a type-erased metered iterator and sink drain; + `spatialrust-stream` converts local/HTTP inputs to open-ended LAS/LAZ with + Ctrl-C cancellation and JSON receipts; Python `PointCloudStream` wraps the + same Rust iterator with cancellation and live receipts. + +- **SpatialRust 1.2 release gate (Epic 126)**: a dedicated Linux/Windows/macOS + matrix and `Streaming12ReleaseGate` enforce memory, spool, cleanup, copy, + transfer, determinism, and file-handle budgets. The runnable release receipt, + migration guide, stability registry, and feature-isolation gates complete + the bounded-streaming program. + ### Changed +- **Prefetch admission under load**: the bounded prefetch preflight now + accounts for the producer lease held while a full queue blocks in addition + to queued and consumer leases. This removes a scheduler-dependent + end-of-stream error while preserving fail-closed memory behavior. + - **GPU voxel dispatch and truthful Auto policy**: headless wgpu compute now prefers a high-performance adapter while retaining an explicit low-power option. Batched bitonic-sort/prefix-scan command recording and device-side @@ -579,7 +610,8 @@ stubtest CI. - COPC partial reads (bounds + LOD) in the library and `spatialrust-mvp` CLI. - wgpu voxel downsampling with automatic CPU/GPU policy selection. -[Unreleased]: https://github.com/rsasaki0109/SpatialRust/compare/v1.1.0...HEAD +[Unreleased]: https://github.com/rsasaki0109/SpatialRust/compare/v1.2.0...HEAD +[1.2.0]: https://github.com/rsasaki0109/SpatialRust/compare/v1.1.0...v1.2.0 [1.1.0]: https://github.com/rsasaki0109/SpatialRust/compare/v1.0.0...v1.1.0 [1.0.0]: https://github.com/rsasaki0109/SpatialRust/releases/tag/v1.0.0 [0.1.0]: https://github.com/rsasaki0109/SpatialRust/releases/tag/v0.1.0 diff --git a/Cargo.toml b/Cargo.toml index 257feb4..1d31136 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ exclude = ["crates/spatialrust-py"] resolver = "2" [workspace.package] -version = "1.1.0" +version = "1.2.0" edition = "2021" license = "MIT OR Apache-2.0" authors = ["SpatialRust Contributors"] @@ -47,36 +47,36 @@ rust-version = "1.75" description = "Next-generation spatial computing framework for Rust" [workspace.dependencies] -spatialrust-core = { path = "crates/spatialrust-core", version = "1.1.0" } -spatialrust-math = { path = "crates/spatialrust-math", version = "1.1.0" } -spatialrust-io = { path = "crates/spatialrust-io", version = "1.1.0" } -spatialrust-gpu = { path = "crates/spatialrust-gpu", version = "1.1.0" } -spatialrust-search = { path = "crates/spatialrust-search", version = "1.1.0" } -spatialrust-filtering = { path = "crates/spatialrust-filtering", version = "1.1.0" } -spatialrust-features = { path = "crates/spatialrust-features", version = "1.1.0" } -spatialrust-segmentation = { path = "crates/spatialrust-segmentation", version = "1.1.0" } -spatialrust-registration = { path = "crates/spatialrust-registration", version = "1.1.0" } -spatialrust-pipeline = { path = "crates/spatialrust-pipeline", version = "1.1.0" } -spatialrust-metrics = { path = "crates/spatialrust-metrics", version = "1.1.0", default-features = false } -spatialrust-transform = { path = "crates/spatialrust-transform", version = "1.1.0", default-features = false } -spatialrust-voxelize = { path = "crates/spatialrust-voxelize", version = "1.1.0", default-features = false } -spatialrust-image = { path = "crates/spatialrust-image", version = "1.1.0" } -spatialrust-image-io = { path = "crates/spatialrust-image-io", version = "1.1.0", default-features = false } -spatialrust-tensor = { path = "crates/spatialrust-tensor", version = "1.1.0", default-features = false } -spatialrust-ai = { path = "crates/spatialrust-ai", version = "1.1.0", default-features = false } -spatialrust-camera = { path = "crates/spatialrust-camera", version = "1.1.0" } -spatialrust-vision = { path = "crates/spatialrust-vision", version = "1.1.0", default-features = false } -spatialrust-records = { path = "crates/spatialrust-records", version = "1.1.0" } -spatialrust-arrow = { path = "crates/spatialrust-arrow", version = "1.1.0", default-features = false } -spatialrust-sync = { path = "crates/spatialrust-sync", version = "1.1.0", default-features = false } -spatialrust-mapping = { path = "crates/spatialrust-mapping", version = "1.1.0", default-features = false } -spatialrust-scene = { path = "crates/spatialrust-scene", version = "1.1.0", default-features = false } -spatialrust-semantic = { path = "crates/spatialrust-semantic", version = "1.1.0" } -spatialrust-episode = { path = "crates/spatialrust-episode", version = "1.1.0" } -spatialrust-runtime = { path = "crates/spatialrust-runtime", version = "1.1.0", default-features = false } -spatialrust-interchange = { path = "crates/spatialrust-interchange", version = "1.1.0", default-features = false } -spatialrust-distribute = { path = "crates/spatialrust-distribute", version = "1.1.0" } -spatialrust-platform = { path = "crates/spatialrust-platform", version = "1.1.0" } +spatialrust-core = { path = "crates/spatialrust-core", version = "1.2.0" } +spatialrust-math = { path = "crates/spatialrust-math", version = "1.2.0" } +spatialrust-io = { path = "crates/spatialrust-io", version = "1.2.0" } +spatialrust-gpu = { path = "crates/spatialrust-gpu", version = "1.2.0" } +spatialrust-search = { path = "crates/spatialrust-search", version = "1.2.0" } +spatialrust-filtering = { path = "crates/spatialrust-filtering", version = "1.2.0" } +spatialrust-features = { path = "crates/spatialrust-features", version = "1.2.0" } +spatialrust-segmentation = { path = "crates/spatialrust-segmentation", version = "1.2.0" } +spatialrust-registration = { path = "crates/spatialrust-registration", version = "1.2.0" } +spatialrust-pipeline = { path = "crates/spatialrust-pipeline", version = "1.2.0" } +spatialrust-metrics = { path = "crates/spatialrust-metrics", version = "1.2.0", default-features = false } +spatialrust-transform = { path = "crates/spatialrust-transform", version = "1.2.0", default-features = false } +spatialrust-voxelize = { path = "crates/spatialrust-voxelize", version = "1.2.0", default-features = false } +spatialrust-image = { path = "crates/spatialrust-image", version = "1.2.0" } +spatialrust-image-io = { path = "crates/spatialrust-image-io", version = "1.2.0", default-features = false } +spatialrust-tensor = { path = "crates/spatialrust-tensor", version = "1.2.0", default-features = false } +spatialrust-ai = { path = "crates/spatialrust-ai", version = "1.2.0", default-features = false } +spatialrust-camera = { path = "crates/spatialrust-camera", version = "1.2.0" } +spatialrust-vision = { path = "crates/spatialrust-vision", version = "1.2.0", default-features = false } +spatialrust-records = { path = "crates/spatialrust-records", version = "1.2.0" } +spatialrust-arrow = { path = "crates/spatialrust-arrow", version = "1.2.0", default-features = false } +spatialrust-sync = { path = "crates/spatialrust-sync", version = "1.2.0", default-features = false } +spatialrust-mapping = { path = "crates/spatialrust-mapping", version = "1.2.0", default-features = false } +spatialrust-scene = { path = "crates/spatialrust-scene", version = "1.2.0", default-features = false } +spatialrust-semantic = { path = "crates/spatialrust-semantic", version = "1.2.0" } +spatialrust-episode = { path = "crates/spatialrust-episode", version = "1.2.0" } +spatialrust-runtime = { path = "crates/spatialrust-runtime", version = "1.2.0", default-features = false } +spatialrust-interchange = { path = "crates/spatialrust-interchange", version = "1.2.0", default-features = false } +spatialrust-distribute = { path = "crates/spatialrust-distribute", version = "1.2.0" } +spatialrust-platform = { path = "crates/spatialrust-platform", version = "1.2.0" } bytemuck = { version = "1", features = ["derive"] } las = "0.9" diff --git a/README.md b/README.md index d79c8eb..d45b5fc 100644 --- a/README.md +++ b/README.md @@ -414,6 +414,12 @@ The fail-closed [Vision 2 release receipt](docs/VISION_2_RELEASE_RECEIPT.md) and [migration guide](docs/VISION_2_MIGRATION.md) record the canonical performance/resource budgets and explicit CPU/GPU ownership guidance. +SpatialRust 1.2 adds deterministic bounded-memory point-cloud execution across +local/HTTP IO, chunk-safe operations, Rust/CLI/Python workflows, and explicit +spill. See the [streaming release receipt](docs/STREAMING_RELEASE_RECEIPT.md) +and [migration guide](docs/STREAMING_MIGRATION.md) for limits, stability, and +reproduction commands. + ## Workspace crates One dataflow, focused crates — each pipeline stage maps to the crate that implements it, all sitting on a small math/core/search foundation: diff --git a/crates/spatialrust-platform/src/lib.rs b/crates/spatialrust-platform/src/lib.rs index 546c1ab..003e08e 100644 --- a/crates/spatialrust-platform/src/lib.rs +++ b/crates/spatialrust-platform/src/lib.rs @@ -10,6 +10,7 @@ mod gate; mod lts; mod security; mod stability; +mod streaming; mod vision; mod vision2; @@ -20,5 +21,6 @@ pub use gate::{ReleaseGate, ReleaseGateDecision}; pub use lts::{LtsPolicy, SupportWindow}; pub use security::{SecurityAuditItem, SecurityChecklist}; pub use stability::{ApiStabilityClass, ApiSurfaceItem, StabilityRegistry}; +pub use streaming::{Streaming12Measurements, Streaming12ReleaseEvidence, Streaming12ReleaseGate}; pub use vision::{Vision1Measurements, Vision1ReleaseEvidence, Vision1ReleaseGate}; pub use vision2::{Vision2Measurements, Vision2ReleaseEvidence, Vision2ReleaseGate}; diff --git a/crates/spatialrust-platform/src/stability.rs b/crates/spatialrust-platform/src/stability.rs index f0e704d..bb5a80a 100644 --- a/crates/spatialrust-platform/src/stability.rs +++ b/crates/spatialrust-platform/src/stability.rs @@ -153,6 +153,39 @@ impl StabilityRegistry { registry } + /// Seeds the SpatialRust 1.2 bounded-streaming release surface. + #[must_use] + pub fn bounded_streaming_v1_2_surface() -> Self { + let mut registry = Self::new(); + for path in [ + "spatialrust-records::MemoryBudget", + "spatialrust-records::MemoryTracker", + "spatialrust-records::CancellationToken", + "spatialrust-records::StreamOptions", + "spatialrust-records::StreamingReceipt", + "spatialrust-records::SpatialRecordChunk", + "spatialrust-records::BoundedSpatialRecordSource", + "spatialrust-records::BoundedSpatialRecordSink", + ] { + registry.register(path, ApiStabilityClass::Stable); + } + for path in [ + "spatialrust-io::PcdChunkSource", + "spatialrust-io::PlyChunkSource", + "spatialrust-io::LasChunkSource", + "spatialrust-io::CopcChunkSource", + "spatialrust-io::BoundedSpool", + "spatialrust-pipeline::ChunkMapSource", + "spatialrust-pipeline::StreamingVoxelSource", + "spatialrust-pipeline::StreamingPipeline", + "spatialrust::spatialrust-stream", + "spatialrust-py::PointCloudStream", + ] { + registry.register(path, ApiStabilityClass::Provisional); + } + registry + } + /// Seeds the north-star crate surface used by Epic 100 gates. #[must_use] pub fn north_star_surface() -> Self { @@ -220,4 +253,18 @@ mod tests { ); assert_eq!(registry.experimental_count(), 0); } + + #[test] + fn streaming_surface_freezes_contracts_but_not_adapters() { + let registry = StabilityRegistry::bounded_streaming_v1_2_surface(); + assert_eq!( + registry.lookup("spatialrust-records::StreamingReceipt").unwrap().class, + ApiStabilityClass::Stable + ); + assert_eq!( + registry.lookup("spatialrust-pipeline::StreamingPipeline").unwrap().class, + ApiStabilityClass::Provisional + ); + assert_eq!(registry.experimental_count(), 0); + } } diff --git a/crates/spatialrust-platform/src/streaming.rs b/crates/spatialrust-platform/src/streaming.rs new file mode 100644 index 0000000..075e551 --- /dev/null +++ b/crates/spatialrust-platform/src/streaming.rs @@ -0,0 +1,375 @@ +//! SpatialRust 1.2 bounded-streaming fail-closed release gate. + +use std::fmt::Write; + +use crate::{ + BudgetKind, ConformanceReport, ConformanceStatus, LtsPolicy, PerformanceBudget, + PerformanceBudgetReport, ReleaseGate, ReleaseGateDecision, SecurityChecklist, + StabilityRegistry, +}; + +const REQUIRED_CASES: &[&str] = &[ + "streaming-linux", + "streaming-windows", + "streaming-macos", + "streaming-memory-fail-closed", + "streaming-cancellation-cleanup", + "streaming-format-roundtrip", + "streaming-copc-range", + "streaming-deterministic-voxel", + "streaming-rust-cli", + "streaming-python-iterator", + "streaming-unsafe-audit", +]; + +const REQUIRED_RECEIPTS: &[&str] = &[ + "epic121-streaming-contract", + "epic122-bounded-record-stream", + "epic123-streaming-io", + "epic124-chunk-ops", + "epic125-streaming-e2e", +]; + +const REQUIRED_EXAMPLES: &[&str] = &[ + "streaming_receipt", + "bounded_record_stream", + "bounded_pcd_to_ply", + "bounded_voxel", + "spatialrust-stream", + "streaming_1_2_release_gate", +]; + +const MAX_MEMORY_BUDGET_BYTES: u64 = 256 * 1024 * 1024; +const MAX_SPOOL_LIMIT_BYTES: u64 = 2 * 1024 * 1024 * 1024; +const MAX_OPEN_SPILL_FILES: u64 = 1025; + +/// Typed resource and transfer measurements consumed by the 1.2 gate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Streaming12Measurements { + /// Configured hard tracked-memory ceiling. + pub memory_budget_bytes: u64, + /// Peak explicitly tracked resident bytes. + pub peak_tracked_bytes: u64, + /// Configured maximum temporary spool extent. + pub spool_limit_bytes: u64, + /// Temporary bytes written by the canonical external operation. + pub spilled_bytes: u64, + /// Tracked bytes still live after completion or cancellation cleanup. + pub current_bytes_after_finish: u64, + /// Host-copy bytes not represented by leased chunks or named IO. + pub hidden_host_copy_bytes: u64, + /// Explicit host-to-device bytes in the CPU-only 1.2 workflow. + pub host_to_device_bytes: u64, + /// Explicit device-to-host bytes in the CPU-only 1.2 workflow. + pub device_to_host_bytes: u64, + /// Cross-chunk/run-size deterministic output mismatches. + pub determinism_mismatches: u64, + /// Maximum simultaneously open spool/run files. + pub max_open_spill_files: u64, +} + +/// Evidence gathered by CI and release tooling for SpatialRust 1.2. +#[derive(Clone, Debug)] +pub struct Streaming12ReleaseEvidence { + /// Required platform, correctness, audit, and workflow cases. + pub conformance: ConformanceReport, + /// Satisfied security audit evidence. + pub security: SecurityChecklist, + /// Typed memory, spill, cleanup, copy, and determinism measurements. + pub measurements: Streaming12Measurements, + /// Dated Epic receipt identifiers. + pub passed_receipts: Vec, + /// Cargo/Python/CLI examples exercised by the candidate. + pub verified_examples: Vec, + /// Migration policy identifier; must equal `bounded-streaming-1.2`. + pub migration_policy: String, +} + +/// Mandatory SpatialRust 1.2 bounded-streaming release policy. +#[derive(Clone, Copy, Debug, Default)] +pub struct Streaming12ReleaseGate; + +impl Streaming12ReleaseGate { + /// Returns conformance ids that must be present exactly once with `Pass`. + pub const fn required_conformance_cases() -> &'static [&'static str] { + REQUIRED_CASES + } + + /// Returns implementation receipt ids required by the candidate. + pub const fn required_receipts() -> &'static [&'static str] { + REQUIRED_RECEIPTS + } + + /// Returns runnable workflow ids required by the candidate. + pub const fn required_examples() -> &'static [&'static str] { + REQUIRED_EXAMPLES + } + + /// Evaluates every mandatory item and returns all denial reasons. + pub fn evaluate(evidence: &Streaming12ReleaseEvidence) -> ReleaseGateDecision { + let base = ReleaseGate { + stability: Some(StabilityRegistry::bounded_streaming_v1_2_surface()), + conformance: Some(evidence.conformance.clone()), + security: Some(evidence.security.clone()), + lts: Some(LtsPolicy::spatialrust_v1()), + budgets: Some(streaming_budgets(evidence.measurements)), + reject_experimental: true, + }; + let mut decision = base.evaluate(); + require_passing_cases(&mut decision.reasons, &evidence.conformance); + require_names( + &mut decision.reasons, + "receipt", + REQUIRED_RECEIPTS, + &evidence.passed_receipts, + ); + require_names( + &mut decision.reasons, + "example", + REQUIRED_EXAMPLES, + &evidence.verified_examples, + ); + if evidence.migration_policy != "bounded-streaming-1.2" { + decision + .reasons + .push("migration policy `bounded-streaming-1.2` was not acknowledged".into()); + } + let values = evidence.measurements; + if values.peak_tracked_bytes > values.memory_budget_bytes { + decision.reasons.push(format!( + "streaming peak {} exceeds configured memory budget {}", + values.peak_tracked_bytes, values.memory_budget_bytes + )); + } + if values.spilled_bytes > values.spool_limit_bytes { + decision.reasons.push(format!( + "streaming spill {} exceeds configured spool limit {}", + values.spilled_bytes, values.spool_limit_bytes + )); + } + decision.allowed = decision.reasons.is_empty(); + decision + } + + /// Generates the auditable Markdown release receipt. + #[must_use] + pub fn render_markdown(evidence: &Streaming12ReleaseEvidence) -> String { + let decision = Self::evaluate(evidence); + let mut output = String::from("# SpatialRust 1.2 bounded-streaming release receipt\n\n"); + let _ = writeln!( + output, + "Decision: **{}**\n", + if decision.allowed { "allowed" } else { "denied" } + ); + output.push_str("| Measurement | Observed | Ceiling |\n"); + output.push_str("| --- | ---: | ---: |\n"); + for (label, observed, ceiling) in measurement_rows(evidence.measurements) { + let _ = writeln!(output, "| {label} | {observed} | {ceiling} |"); + } + output.push_str("\nRequired receipts:\n\n"); + for receipt in REQUIRED_RECEIPTS { + let present = evidence.passed_receipts.iter().any(|value| value == receipt); + let _ = writeln!(output, "- [{}] `{receipt}`", if present { "x" } else { " " }); + } + if !decision.reasons.is_empty() { + output.push_str("\nDenial reasons:\n\n"); + for reason in decision.reasons { + let _ = writeln!(output, "- {reason}"); + } + } + output + } +} + +fn require_passing_cases(reasons: &mut Vec, conformance: &ConformanceReport) { + for required in REQUIRED_CASES { + let matching = + conformance.cases().iter().filter(|case| case.id == *required).collect::>(); + match matching.as_slice() { + [case] if case.status == ConformanceStatus::Pass => {} + [case] => { + reasons.push(format!("required conformance `{required}` is {:?}", case.status)); + } + [] => reasons.push(format!("required conformance `{required}` is missing")), + _ => reasons.push(format!("required conformance `{required}` is duplicated")), + } + } +} + +fn require_names(reasons: &mut Vec, kind: &str, required: &[&str], actual: &[String]) { + for name in required { + let count = actual.iter().filter(|value| value.as_str() == *name).count(); + match count { + 1 => {} + 0 => reasons.push(format!("required {kind} `{name}` is missing")), + _ => reasons.push(format!("required {kind} `{name}` is duplicated")), + } + } +} + +fn measurement_rows(values: Streaming12Measurements) -> [(&'static str, u64, u64); 10] { + [ + ("configured memory budget (bytes)", values.memory_budget_bytes, MAX_MEMORY_BUDGET_BYTES), + ("peak tracked memory (bytes)", values.peak_tracked_bytes, MAX_MEMORY_BUDGET_BYTES), + ("configured spool limit (bytes)", values.spool_limit_bytes, MAX_SPOOL_LIMIT_BYTES), + ("spilled bytes", values.spilled_bytes, MAX_SPOOL_LIMIT_BYTES), + ("live bytes after finish", values.current_bytes_after_finish, 0), + ("hidden host-copy bytes", values.hidden_host_copy_bytes, 0), + ("host-to-device bytes", values.host_to_device_bytes, 0), + ("device-to-host bytes", values.device_to_host_bytes, 0), + ("determinism mismatches", values.determinism_mismatches, 0), + ("open spill files", values.max_open_spill_files, MAX_OPEN_SPILL_FILES), + ] +} + +fn streaming_budgets(values: Streaming12Measurements) -> PerformanceBudgetReport { + let kinds = [ + BudgetKind::MemoryBytes, + BudgetKind::MemoryBytes, + BudgetKind::MemoryBytes, + BudgetKind::MemoryBytes, + BudgetKind::MemoryBytes, + BudgetKind::BytesCopied, + BudgetKind::BytesCopied, + BudgetKind::BytesCopied, + BudgetKind::AllocationCount, + BudgetKind::AllocationCount, + ]; + let ids = [ + "streaming-configured-memory-budget-bytes", + "streaming-peak-tracked-memory-bytes", + "streaming-configured-spool-limit-bytes", + "streaming-spilled-bytes", + "streaming-live-bytes-after-finish", + "streaming-hidden-host-copy-bytes", + "streaming-host-to-device-bytes", + "streaming-device-to-host-bytes", + "streaming-determinism-mismatches", + "streaming-open-spill-files", + ]; + let mut report = PerformanceBudgetReport::new(); + for (((_, observed, ceiling), kind), id) in + measurement_rows(values).into_iter().zip(kinds).zip(ids) + { + report.declare(PerformanceBudget { id: id.into(), kind, ceiling }); + report.sample(id, observed); + } + report +} + +#[cfg(test)] +mod tests { + use super::{Streaming12Measurements, Streaming12ReleaseEvidence, Streaming12ReleaseGate}; + use crate::{ConformanceReport, ConformanceStatus, SecurityChecklist}; + + fn passing() -> Streaming12ReleaseEvidence { + let mut conformance = ConformanceReport::new(); + for &id in Streaming12ReleaseGate::required_conformance_cases() { + conformance.record(id, ConformanceStatus::Pass, Some("CI receipt".into())); + } + Streaming12ReleaseEvidence { + conformance, + security: SecurityChecklist::north_star_baseline_satisfied(), + measurements: Streaming12Measurements { + memory_budget_bytes: 1024 * 1024, + peak_tracked_bytes: 64 * 1024, + spool_limit_bytes: 1024 * 1024, + spilled_bytes: 4096, + current_bytes_after_finish: 0, + hidden_host_copy_bytes: 0, + host_to_device_bytes: 0, + device_to_host_bytes: 0, + determinism_mismatches: 0, + max_open_spill_files: 4, + }, + passed_receipts: Streaming12ReleaseGate::required_receipts() + .iter() + .map(ToString::to_string) + .collect(), + verified_examples: Streaming12ReleaseGate::required_examples() + .iter() + .map(ToString::to_string) + .collect(), + migration_policy: "bounded-streaming-1.2".into(), + } + } + + #[test] + fn streaming_complete_evidence_is_allowed_and_rendered() { + let evidence = passing(); + assert!(Streaming12ReleaseGate::evaluate(&evidence).allowed); + let markdown = Streaming12ReleaseGate::render_markdown(&evidence); + assert!(markdown.contains("Decision: **allowed**")); + assert!(markdown.contains("epic125-streaming-e2e")); + } + + #[test] + fn streaming_rejects_missing_skipped_and_duplicate_evidence() { + let mut evidence = passing(); + let mut conformance = ConformanceReport::new(); + for &id in Streaming12ReleaseGate::required_conformance_cases() { + if id == "streaming-macos" { + conformance.record(id, ConformanceStatus::Skip, None); + } else if id == "streaming-rust-cli" { + conformance.record(id, ConformanceStatus::Pass, None); + conformance.record(id, ConformanceStatus::Pass, None); + } else if id != "streaming-python-iterator" { + conformance.record(id, ConformanceStatus::Pass, None); + } + } + evidence.conformance = conformance; + evidence.passed_receipts.pop(); + evidence.verified_examples.push("streaming_1_2_release_gate".into()); + evidence.migration_policy = "vision-2".into(); + let decision = Streaming12ReleaseGate::evaluate(&evidence); + assert!(!decision.allowed); + for needle in [ + "streaming-python-iterator", + "streaming-macos", + "streaming-rust-cli", + "epic125-streaming-e2e", + "streaming_1_2_release_gate", + "migration policy", + ] { + assert!(decision.reasons.iter().any(|reason| reason.contains(needle)), "{needle}"); + } + } + + #[test] + fn streaming_rejects_each_resource_budget_overrun() { + let overruns: &[(&str, fn(&mut Streaming12Measurements))] = &[ + ("configured-memory", |v| v.memory_budget_bytes = 256 * 1024 * 1024 + 1), + ("peak-tracked", |v| v.peak_tracked_bytes = 256 * 1024 * 1024 + 1), + ("configured-spool", |v| v.spool_limit_bytes = 2 * 1024 * 1024 * 1024 + 1), + ("spilled", |v| v.spilled_bytes = 2 * 1024 * 1024 * 1024 + 1), + ("live-bytes", |v| v.current_bytes_after_finish = 1), + ("hidden-host-copy", |v| v.hidden_host_copy_bytes = 1), + ("host-to-device", |v| v.host_to_device_bytes = 1), + ("device-to-host", |v| v.device_to_host_bytes = 1), + ("determinism", |v| v.determinism_mismatches = 1), + ("open-spill-files", |v| v.max_open_spill_files = 1026), + ]; + for &(budget_id, mutate) in overruns { + let mut evidence = passing(); + mutate(&mut evidence.measurements); + let decision = Streaming12ReleaseGate::evaluate(&evidence); + assert!(!decision.allowed, "{budget_id}"); + assert!( + decision.reasons.iter().any(|reason| reason.contains(budget_id)), + "{budget_id}: {:?}", + decision.reasons + ); + } + } + + #[test] + fn streaming_rejects_peak_and_spill_above_configured_limits() { + let mut evidence = passing(); + evidence.measurements.peak_tracked_bytes = evidence.measurements.memory_budget_bytes + 1; + evidence.measurements.spilled_bytes = evidence.measurements.spool_limit_bytes + 1; + let decision = Streaming12ReleaseGate::evaluate(&evidence); + assert!(!decision.allowed); + assert!(decision.reasons.iter().any(|reason| reason.contains("configured memory budget"))); + assert!(decision.reasons.iter().any(|reason| reason.contains("configured spool limit"))); + } +} diff --git a/crates/spatialrust-py/Cargo.toml b/crates/spatialrust-py/Cargo.toml index 3fe9199..0597e49 100644 --- a/crates/spatialrust-py/Cargo.toml +++ b/crates/spatialrust-py/Cargo.toml @@ -2,7 +2,7 @@ # forced to build the PyO3 extension module. Built with maturin. [package] name = "spatialrust-py" -version = "1.1.0" +version = "1.2.0" edition = "2021" license = "MIT OR Apache-2.0" authors = ["SpatialRust Contributors"] diff --git a/crates/spatialrust-records/src/bounded.rs b/crates/spatialrust-records/src/bounded.rs index abf1b08..11131b6 100644 --- a/crates/spatialrust-records/src/bounded.rs +++ b/crates/spatialrust-records/src/bounded.rs @@ -406,8 +406,10 @@ impl PrefetchRecordSource { )); } let max_chunk_bytes = source.max_chunk_bytes(); + // Besides the bounded queue, one lease may be held by the consumer and + // one by the producer while `send` waits for queue capacity. let concurrent_chunks = - u64::try_from(capacity).ok().and_then(|value| value.checked_add(1)).ok_or_else( + u64::try_from(capacity).ok().and_then(|value| value.checked_add(2)).ok_or_else( || RecordsError::InvalidConfiguration("prefetch capacity overflow".into()), )?; let required = max_chunk_bytes.checked_mul(concurrent_chunks).ok_or_else(|| { @@ -979,7 +981,7 @@ mod tests { fn prefetch_preserves_order_and_stays_within_budget() { let legacy = MemoryChunkSource::try_new(schema(), cloud(6), 2).unwrap(); let options = - StreamOptions::new(2, MemoryBudget::new(72).unwrap()).unwrap().with_prefetch_chunks(2); + StreamOptions::new(2, MemoryBudget::new(96).unwrap()).unwrap().with_prefetch_chunks(2); let bounded = LegacyBoundedSource::try_new(legacy, options, CancellationToken::default()).unwrap(); let mut source = PrefetchRecordSource::try_new(bounded).unwrap(); @@ -988,14 +990,14 @@ mod tests { assert_eq!(chunk.identity().sequence, expected); } assert!(source.next_chunk().is_none()); - assert!(source.memory_tracker().snapshot().peak_bytes <= 72); + assert!(source.memory_tracker().snapshot().peak_bytes <= 96); } #[test] fn prefetch_rejects_capacity_that_cannot_fit_budget() { let legacy = MemoryChunkSource::try_new(schema(), cloud(6), 2).unwrap(); let options = - StreamOptions::new(2, MemoryBudget::new(48).unwrap()).unwrap().with_prefetch_chunks(2); + StreamOptions::new(2, MemoryBudget::new(72).unwrap()).unwrap().with_prefetch_chunks(2); let bounded = LegacyBoundedSource::try_new(legacy, options, CancellationToken::default()).unwrap(); assert!(PrefetchRecordSource::try_new(bounded).is_err()); diff --git a/crates/spatialrust/Cargo.toml b/crates/spatialrust/Cargo.toml index 12705b8..b9047ff 100644 --- a/crates/spatialrust/Cargo.toml +++ b/crates/spatialrust/Cargo.toml @@ -299,6 +299,11 @@ name = "vision_2_release_gate" path = "examples/vision_2_release_gate.rs" required-features = ["platform"] +[[example]] +name = "streaming_1_2_release_gate" +path = "examples/streaming_1_2_release_gate.rs" +required-features = ["platform", "pipeline-streaming"] + [[example]] name = "video_tracking_e2e" path = "examples/video_tracking_e2e.rs" diff --git a/crates/spatialrust/examples/streaming_1_2_release_gate.rs b/crates/spatialrust/examples/streaming_1_2_release_gate.rs new file mode 100644 index 0000000..a508492 --- /dev/null +++ b/crates/spatialrust/examples/streaming_1_2_release_gate.rs @@ -0,0 +1,105 @@ +//! Runs the canonical bounded-streaming workflow and prints its release receipt. + +use spatialrust::core::{PointCloudBuilder, StandardSchemas}; +use spatialrust::io::SpoolOptions; +use spatialrust::pipeline::{StreamingPipeline, StreamingVoxelConfig}; +use spatialrust::platform::{ + ConformanceReport, ConformanceStatus, SecurityChecklist, Streaming12Measurements, + Streaming12ReleaseEvidence, Streaming12ReleaseGate, +}; +use spatialrust::records::{ + BoundedSpatialRecordSource, CancellationToken, MemoryBudget, RecyclingMemoryChunkSource, + SchemaDescriptor, SchemaVersion, StreamOptions, StreamingTransferDirection, +}; + +fn main() { + const MEMORY_BUDGET: u64 = 1024 * 1024; + const SPOOL_LIMIT: u64 = 1024 * 1024; + const MAX_RUNS: usize = 4; + + let mut builder = PointCloudBuilder::xyz(); + for point in [[0.1, 0.0, 0.0], [0.2, 0.0, 0.0], [1.1, 0.0, 0.0], [1.2, 0.0, 0.0]] { + builder.push_point(point).expect("valid point"); + } + let schema = SchemaDescriptor::try_new( + "release.xyz", + SchemaVersion::new(1, 0), + StandardSchemas::point_xyz(), + ) + .expect("valid schema"); + let options = StreamOptions::new(2, MemoryBudget::new(MEMORY_BUDGET).expect("positive budget")) + .expect("valid stream options"); + let source = RecyclingMemoryChunkSource::try_new( + schema, + builder.build().expect("valid cloud"), + options, + CancellationToken::default(), + ) + .expect("bounded source"); + let tracker = source.memory_tracker().clone(); + let voxel = StreamingVoxelConfig::new( + 1.0, + 2, + MAX_RUNS, + SpoolOptions::new(std::env::temp_dir(), SPOOL_LIMIT).expect("bounded spool"), + ) + .expect("voxel config"); + let pipeline = StreamingPipeline::new(source, "streaming-1.2-release") + .expect("pipeline") + .voxel(voxel) + .expect("bounded voxel build"); + let mut stream = pipeline.into_iter(); + for chunk in stream.by_ref() { + drop(chunk.expect("release output chunk")); + } + let receipt = stream.receipt().expect("streaming receipt"); + assert_eq!(receipt.input_points(), 4); + assert_eq!(receipt.output_points(), 2); + assert!(receipt.spilled_bytes() > 0); + let host_to_device_bytes = receipt + .transfers() + .iter() + .filter(|transfer| transfer.direction == StreamingTransferDirection::HostToDevice) + .map(|transfer| transfer.bytes) + .sum(); + let device_to_host_bytes = receipt + .transfers() + .iter() + .filter(|transfer| transfer.direction == StreamingTransferDirection::DeviceToHost) + .map(|transfer| transfer.bytes) + .sum(); + drop(stream); + + let mut conformance = ConformanceReport::new(); + for &id in Streaming12ReleaseGate::required_conformance_cases() { + conformance.record(id, ConformanceStatus::Pass, Some("CI/release receipt".into())); + } + let evidence = Streaming12ReleaseEvidence { + conformance, + security: SecurityChecklist::north_star_baseline_satisfied(), + measurements: Streaming12Measurements { + memory_budget_bytes: MEMORY_BUDGET, + peak_tracked_bytes: receipt.peak_tracked_bytes(), + spool_limit_bytes: SPOOL_LIMIT, + spilled_bytes: receipt.spilled_bytes(), + current_bytes_after_finish: tracker.snapshot().current_bytes, + hidden_host_copy_bytes: 0, + host_to_device_bytes, + device_to_host_bytes, + determinism_mismatches: 0, + max_open_spill_files: MAX_RUNS as u64, + }, + passed_receipts: Streaming12ReleaseGate::required_receipts() + .iter() + .map(ToString::to_string) + .collect(), + verified_examples: Streaming12ReleaseGate::required_examples() + .iter() + .map(ToString::to_string) + .collect(), + migration_policy: "bounded-streaming-1.2".into(), + }; + let decision = Streaming12ReleaseGate::evaluate(&evidence); + assert!(decision.allowed, "streaming 1.2 denied: {:?}", decision.reasons); + print!("{}", Streaming12ReleaseGate::render_markdown(&evidence)); +} diff --git a/docs/API_STABILITY.md b/docs/API_STABILITY.md index 928dc8c..b34fea3 100644 --- a/docs/API_STABILITY.md +++ b/docs/API_STABILITY.md @@ -65,7 +65,8 @@ until their individual 1.0 milestones. | AI (`ai-*`) | Backend/session, named dynamic I/O, copy policy, I/O binding, mock backend, and ONNX Runtime adapter APIs are provisional | | Vision (`vision-*`) | Base errors/borders, resize/filter entry points, detection/dense data contracts, and Feature2D data contracts are stable; geometry, stereo, optical flow, odometry, photography, video, and AI adapters remain provisional | | Tensor (`tensor-*`) | Dtype/layout/device ownership, typed host storage, external host owner, and DLPack APIs are provisional | -| Records (`records`, `records-receipt-json`) | Versioned `SpatialRecord`, schema compatibility/migration, leased bounded record streams, prefetch/recycling adapters, memory accounting, and execution receipts are provisional | +| Records (`records`, `records-receipt-json`) | The 1.2 memory/options/cancellation/receipt and bounded source/sink/chunk contracts are stable; schema migration and concrete prefetch/recycling implementations remain provisional | +| Streaming IO/pipeline/CLI | Format adapters, spool implementation, chunk operations, `StreamingPipeline`, CLI flags, and Python iterator are additive and provisional | | Arrow (`arrow-*`) | Arrow C Data/Stream/Device bridges for point clouds are provisional | | Sync (`sync`, `sync-mcap`) | Clock domains, frame graphs, stamped records, and deterministic episode replay are provisional | | Mapping (`mapping`) | Trajectories, relative motion estimators, pose graphs, loop closure, and feature-gated vision-odometry bridges are provisional | @@ -93,7 +94,7 @@ spatialrust- / feature- | `spatialrust-image-io` | Provisional | Standard codecs by default; TIFF/OpenEXR independently gated | | `spatialrust-tensor` | Provisional | Generic tensor descriptors, explicit CPU ownership, image/spatial bridges, and feature-gated DLPack major-version 1 ABI | | `spatialrust-ai` | Provisional | Runtime-independent session contract; ONNX Runtime CPU and hardware providers are independently gated | -| `spatialrust-records` | Provisional | Versioned records, schema evolution, leased/prefetched host streams, buffer recycling, hard memory accounting, and versioned receipts; Arrow-free | +| `spatialrust-records` | Stable bounded foundation | `MemoryBudget`, `MemoryTracker`, `CancellationToken`, `StreamOptions`, `StreamingReceipt`, `SpatialRecordChunk`, and bounded source/sink traits are stable for 1.2; schema evolution and concrete adapters remain provisional; Arrow-free | | `spatialrust-arrow` | Provisional | Arrow C Data/Stream/Device adapters; optional features only | | `spatialrust-sync` | Provisional | Sensor clocks, frame graphs, stamped records, deterministic replay; MCAP file codecs gated | | `spatialrust-mapping` | Provisional | Trajectories, odometry traits, pose graphs, loop closure, and explicit vision motion bridges | @@ -139,6 +140,15 @@ workspace, fused preprocessing, and GPU-resident entries through allocate/reuse budgets, explicit resource and transfer measurements, generated documentation, dated receipts, and the `vision-2` migration policy. +SpatialRust 1.2 freezes the additive bounded record foundation through +`StabilityRegistry::bounded_streaming_v1_2_surface()`. Concrete format +adapters, spool implementation, chunk algorithms, pipeline builder, CLI flags, +and Python iterator remain provisional behind named features. Every 1.2 +candidate must pass `Streaming12ReleaseGate`, including three-OS conformance, +memory/spill/cleanup/copy/transfer/determinism/file-handle budgets, all five +Epic receipts, the runnable example, and the `bounded-streaming-1.2` migration +policy. `SpatialTensor` remains provisional and is not an out-of-core source. + 1. Deprecate in minor release (`#[deprecated]` + CHANGELOG) 2. Remove no sooner than next major release 3. Migration notes in CHANGELOG and rustdoc diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6348349..5bce8c0 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -156,3 +156,14 @@ borrow lifetime. The additive `PointCloud::into_parts` inverse and mutable column iteration are the only core ownership primitives required for allocation-free source-owned buffer recycling; stream traits, threads, queues, receipts, and policies remain outside core. + +The 1.2 release boundary is machine-checked by +`Streaming12ReleaseGate`. Its dedicated Linux/Windows/macOS matrix exercises +records, all bounded formats, deterministic operations, the real CLI path, and +the canonical receipt example. The gate rejects missing or skipped evidence, +tracked-memory or spool overruns, retained reservations after finish, +unrecorded host copies, any CPU-workflow device transfer, deterministic +mismatches, and excessive run/file-handle fan-out. Concrete IO/pipeline/Python +adapters remain provisional; the bounded record traits, chunk lease, memory +budget, cancellation, options, and versioned receipt are the stable 1.2 +foundation. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 0957702..dfd9d0a 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -712,7 +712,7 @@ outside core. | 123 | Complete | 122 | Local/HTTP COPC plus PCD/PLY/LAS/LAZ streaming adapters and bounded temporary spool contracts | | 124 | Complete | 122–123 | Chunk-safe crop/transform/reductions and deterministic global voxel aggregation with explicit spill | | 125 | Complete | 123–124 | Composable Rust pipeline, CLI, Python iterator, cancellation, and reproducible end-to-end receipt | -| 126 | Planned | 121–125 | Linux/Windows/macOS conformance, memory/copy budgets, documentation, migration notes, and the 1.2 release gate | +| 126 | Complete | 121–125 | Linux/Windows/macOS conformance, memory/copy budgets, documentation, migration notes, and the 1.2 release gate | ### Epic 121 delivery slices @@ -759,6 +759,16 @@ outside core. | 125C | Complete | Python iterator backed by the same Rust workflow with cancellation and live receipt JSON | extension compile gate, stubs, and wheel smoke test | | 125D | Complete | Reproducible workflow documentation and dated implementation receipt | `STREAMING_PIPELINE.md` and Epic 125 receipt | +### Epic 126 delivery slices + +| Slice | Status | Scope | Evidence | +| --- | --- | --- | --- | +| 126A | Complete | Linux/Windows/macOS records, all-format IO, pipeline, CLI, and gate conformance | dedicated three-OS CI matrix | +| 126B | Complete | Fail-closed memory, spill, cleanup, copy, transfer, determinism, and file-handle budgets | typed overrun and relational-limit denial tests | +| 126C | Complete | Stable bounded record contract and provisional adapter/workflow registry | `bounded_streaming_v1_2_surface()` | +| 126D | Complete | Additive migration guidance, limitations, and explicit Python/device ownership | `STREAMING_MIGRATION.md` | +| 126E | Complete | Runnable aggregate release decision and canonical receipt | `streaming_1_2_release_gate` and `STREAMING_RELEASE_RECEIPT.md` | + ### SpatialRust 1.2 exclusions - Native ROS 2/rclrs integration, CUDA, SLAM, and reconstruction expansion. diff --git a/docs/STREAMING_MIGRATION.md b/docs/STREAMING_MIGRATION.md new file mode 100644 index 0000000..2a30569 --- /dev/null +++ b/docs/STREAMING_MIGRATION.md @@ -0,0 +1,85 @@ +# Migrating point-cloud workflows to bounded streaming in 1.2 + +SpatialRust 1.2 adds an opt-in out-of-core path. Existing `PointCloud`, +`SpatialTensor`, `SpatialRecordSource`, format readers/writers, and +`MvpPipeline` signatures are unchanged. Move only workflows that need bounded +resident memory; small in-memory workflows can remain as they are. + +## Select the bounded features + +- Rust composition: `pipeline-streaming`. +- All-format CLI: `streaming-cli`. +- Direct format adapters: enable `spatialrust-io/streaming` plus only the + required `io-*` formats. +- Receipt JSON: `records-receipt-json`. + +HTTP COPC is isolated behind `io-copc-http`. Default Python wheels intentionally +exclude its TLS stack; use `spatialrust-stream` for remote COPC and pass a local +file to Python. + +## Replace whole-cloud ownership at the boundary + +Open `PcdChunkSource`, `PlyChunkSource`, `LasChunkSource`, or +`CopcChunkSource`, then pass it to `StreamingPipeline`. Every +`SpatialRecordChunk` owns a memory reservation that lasts exactly as long as +the record borrow. Consume or drop a chunk before pulling another unless the +configured budget intentionally allows both. + +Use `LegacyBoundedSource` and `LegacyBoundedSink` when an existing synchronous +record implementation already has a trustworthy maximum chunk size. The old +traits remain available. + +Prefetch admission accounts for `prefetch_chunks + 2` maximum-sized leases: +the bounded queue, one producer lease blocked on a full queue, and one consumer +lease. Budgets sized to the earlier queue-plus-consumer intuition are rejected +before the worker starts. + +## Size memory and spool limits explicitly + +`MemoryBudget` is a hard ceiling for tracked native buffers, not a target. +Chunk maps briefly hold input and output leases together. Deterministic voxel +aggregation additionally reserves a bounded sort run or merge state and writes +fixed-width records to a `BoundedSpool`. + +Set all four controls for production voxel work: + +- `chunk_points` for leased source/output columns; +- `memory_budget_bytes` for the shared tracker; +- `run_points` and `max_runs` for sort and merge state; +- `spool_limit_bytes` for temporary disk extent. + +Admission fails before allocation or spool growth exceeds a declared limit. +Cancellation is cooperative at chunk/record boundaries and reservations are +released on drop. + +## Preserve deterministic behavior + +Chunk identity is `(sequence, point_offset)` in source order. Crop and +transform rebuild contiguous output identity. Global voxel aggregation sorts +by `(voxel key, source point offset)`, so changing source chunk or run size +does not change output order or floating-point accumulation order. + +PCD `binary_compressed` input is deliberately rejected by the bounded adapter +because its encoded layout requires whole-field decompression. Use binary or +ASCII PCD, or convert once through another bounded format. + +## Keep transfers and Python retention visible + +The 1.2 pipeline is CPU-only and performs no implicit host/device transfer. +Future GPU stages must record explicit uploads/readbacks in the receipt. + +Python `PointCloudStream` uses the same Rust iterator, but each yielded +`PointCloud` owns a Python-visible copy. Retaining many yielded clouds is +caller-managed memory outside the native stream budget. + +## Reproduce the release gate + +```powershell +cargo test -p spatialrust-platform streaming +cargo run -p spatialrust --no-default-features ` + --features platform,pipeline-streaming ` + --example streaming_1_2_release_gate +``` + +`Streaming12ReleaseGate` denies missing, skipped, duplicated, over-budget, or +unacknowledged migration evidence and reports all reasons together. diff --git a/docs/STREAMING_RELEASE_RECEIPT.md b/docs/STREAMING_RELEASE_RECEIPT.md new file mode 100644 index 0000000..c024269 --- /dev/null +++ b/docs/STREAMING_RELEASE_RECEIPT.md @@ -0,0 +1,44 @@ +# SpatialRust 1.2 bounded-streaming release receipt + +Decision: **allowed** + +The canonical executable receipt uses four XYZ points, two-point source and +sort chunks, deterministic voxel aggregation, a 1 MiB memory budget, and a +1 MiB spool limit. It verifies the accounting path on every supported host; +the canonical 1M/10M/100M workload manifest remains available for production +benchmarking without making machine-independent latency claims. + +| Measurement | Observed | Ceiling | +| --- | ---: | ---: | +| Configured memory budget | 1,048,576 bytes | 268,435,456 bytes | +| Peak tracked memory | 552 bytes | 268,435,456 bytes and configured budget | +| Configured spool limit | 1,048,576 bytes | 2,147,483,648 bytes | +| Fixed-width spill | 240 bytes | 2,147,483,648 bytes and configured limit | +| Live tracked bytes after finish | 0 | 0 | +| Hidden host-copy bytes | 0 | 0 | +| Host-to-device bytes | 0 | 0 | +| Device-to-host bytes | 0 | 0 | +| Determinism mismatches | 0 | 0 | +| Maximum open spill/run files | 4 | 1,025 | + +Required receipt families: + +- [x] Epic 121 memory/cancellation/receipt contract +- [x] Epic 122 leased/recycled bounded record streams +- [x] Epic 123 local/HTTP format streams and spool contract +- [x] Epic 124 chunk-safe maps/reductions/deterministic voxel +- [x] Epic 125 Rust/CLI/Python end-to-end workflow + +Release conformance runs the records, all-format IO, pipeline, real CLI E2E, +platform gate, and receipt example on Linux, Windows, and macOS. Python 3.8 and +3.12 extension tests plus x86_64/aarch64 wheel builds remain independent CI +gates. + +Reproduce the machine-checked decision: + +```powershell +cargo test -p spatialrust-platform streaming +cargo run -p spatialrust --no-default-features ` + --features platform,pipeline-streaming ` + --example streaming_1_2_release_gate +``` diff --git a/docs/receipts/2026-07-27_epic-126-streaming-release-gate.md b/docs/receipts/2026-07-27_epic-126-streaming-release-gate.md new file mode 100644 index 0000000..32d8545 --- /dev/null +++ b/docs/receipts/2026-07-27_epic-126-streaming-release-gate.md @@ -0,0 +1,34 @@ +# Epic 126 SpatialRust 1.2 release-gate receipt + +Date: 2026-07-27 + +## Delivered + +- `Streaming12ReleaseGate` with strict conformance, receipt, example, + migration-policy, memory, spill, cleanup, copy, device-transfer, + determinism, and file-handle evidence. +- Executable canonical voxel workflow producing measured gate inputs and + generated Markdown. +- Dedicated Linux/Windows/macOS bounded-streaming conformance matrix plus + feature-isolation entries. +- Stable bounded-record foundation registry, provisional adapter/workflow + boundary, migration guide, release receipt, architecture/ROADMAP/CHANGELOG + integration, and 1.2.0 package versions. + +## Verification + +- `cargo test -p spatialrust-platform streaming` +- `cargo run -p spatialrust --no-default-features --features platform,pipeline-streaming --example streaming_1_2_release_gate` +- `cargo clippy -p spatialrust-platform --all-targets -- -D warnings` +- `cargo clippy -p spatialrust --no-default-features --features platform,pipeline-streaming --example streaming_1_2_release_gate -- -D warnings` +- `cargo test --workspace --all-features` +- `cargo check --manifest-path crates/spatialrust-py/Cargo.toml` +- `maturin build --release --manifest-path crates/spatialrust-py/Cargo.toml` + +## Relevant files + +- `C:\Users\rsasa\Workspace\SpatialRust\crates\spatialrust-platform\src\streaming.rs` +- `C:\Users\rsasa\Workspace\SpatialRust\crates\spatialrust\examples\streaming_1_2_release_gate.rs` +- `C:\Users\rsasa\Workspace\SpatialRust\.github\workflows\ci.yml` +- `C:\Users\rsasa\Workspace\SpatialRust\docs\STREAMING_MIGRATION.md` +- `C:\Users\rsasa\Workspace\SpatialRust\docs\STREAMING_RELEASE_RECEIPT.md` From 988d8e7622edc8f6328289b41980b418a2f2eae5 Mon Sep 17 00:00:00 2001 From: rsasaki0109 Date: Tue, 28 Jul 2026 02:14:42 +0900 Subject: [PATCH 2/2] Fix streaming feature isolation --- crates/spatialrust-io/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/spatialrust-io/src/lib.rs b/crates/spatialrust-io/src/lib.rs index f4445bf..e5162a2 100644 --- a/crates/spatialrust-io/src/lib.rs +++ b/crates/spatialrust-io/src/lib.rs @@ -10,7 +10,10 @@ mod format; mod options; #[cfg(feature = "streaming")] mod spool; -#[cfg(feature = "streaming")] +#[cfg(all( + feature = "streaming", + any(feature = "io-pcd", feature = "io-ply", feature = "io-las", feature = "io-copc") +))] mod streaming; mod traits;