diff --git a/Cargo.toml b/Cargo.toml index 6cf267e..3174d0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,6 +109,7 @@ exr = { version = "=1.72.0", default-features = false } tempfile = "3" winit = "0.30" serde_json = "1" +sha2 = "0.10" wasm-bindgen = "0.2" wasm-bindgen-futures = "0.4" js-sys = "0.3" diff --git a/README.md b/README.md index d453ce9..b191a4d 100644 --- a/README.md +++ b/README.md @@ -641,6 +641,19 @@ let cloud = read_point_cloud_file("scan.las")?; write_point_cloud_file("output.ply", &cloud)?; ``` +For datasets on an external SSD, resolve logical input/output paths explicitly +and emit a size/SHA-256 manifest: + +```bash +cargo run -p spatialrust --features mvp --bin spatialrust-mvp -- \ + --input-root /media/sasaki/aiueo/datasets \ + --output-root /media/sasaki/aiueo/spatialrust-results \ + --manifest runs/scan.json boreas/scan.las runs/scan.ply +``` + +See [`docs/EXTERNAL_STORAGE.md`](docs/EXTERNAL_STORAGE.md) for the Python and +bounded-streaming equivalents. + COPC partial read: ```rust diff --git a/crates/spatialrust-io/Cargo.toml b/crates/spatialrust-io/Cargo.toml index 378b9fb..04288bd 100644 --- a/crates/spatialrust-io/Cargo.toml +++ b/crates/spatialrust-io/Cargo.toml @@ -19,6 +19,7 @@ io-copc = ["dep:copc-streaming", "dep:copc-core", "dep:copc-writer", "dep:pollst io-copc-http = ["io-copc", "dep:ureq"] streaming = ["dep:spatialrust-records"] serde = ["spatialrust-core/serde"] +io-manifest = ["dep:serde", "dep:serde_json", "dep:sha2"] [dependencies] spatialrust-core = { workspace = true } @@ -32,8 +33,12 @@ copc-core = { workspace = true, optional = true } copc-writer = { workspace = true, optional = true } pollster = { workspace = true, optional = true } ureq = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } +sha2 = { workspace = true, optional = true } [dev-dependencies] +tempfile.workspace = true [[example]] name = "bounded_pcd_to_ply" diff --git a/crates/spatialrust-io/src/error.rs b/crates/spatialrust-io/src/error.rs index d5da0ce..97d466b 100644 --- a/crates/spatialrust-io/src/error.rs +++ b/crates/spatialrust-io/src/error.rs @@ -57,6 +57,14 @@ pub enum IoError { #[error("streaming io error: {0}")] Streaming(String), + /// Input/output root or dataset path contract failure. + #[error("storage path error: {0}")] + Storage(String), + + /// Dataset manifest serialization or receipt failure. + #[error("manifest error: {0}")] + Manifest(String), + /// Core data model error propagated from `spatialrust-core`. #[error(transparent)] Core(#[from] SpatialError), diff --git a/crates/spatialrust-io/src/lib.rs b/crates/spatialrust-io/src/lib.rs index e5162a2..beee2c6 100644 --- a/crates/spatialrust-io/src/lib.rs +++ b/crates/spatialrust-io/src/lib.rs @@ -7,9 +7,12 @@ mod error; mod format; +#[cfg(feature = "io-manifest")] +mod manifest; mod options; #[cfg(feature = "streaming")] mod spool; +mod storage; #[cfg(all( feature = "streaming", any(feature = "io-pcd", feature = "io-ply", feature = "io-las", feature = "io-copc") @@ -42,9 +45,12 @@ pub use format::{ detect_point_cloud_format, read_point_cloud_file, read_point_cloud_file_with_format, write_point_cloud_file, write_point_cloud_file_with_format, PointCloudFileFormat, }; +#[cfg(feature = "io-manifest")] +pub use manifest::{DatasetManifest, FileReceipt, ReceiptRole, DATASET_MANIFEST_VERSION}; pub use options::{ReadOptions, WriteOptions}; #[cfg(feature = "streaming")] pub use spool::{BoundedSpool, SpoolOptions}; +pub use storage::StorageRoots; pub use traits::{PointReader, PointSink, PointStream, PointWriter}; #[cfg(feature = "io-pcd")] diff --git a/crates/spatialrust-io/src/manifest.rs b/crates/spatialrust-io/src/manifest.rs new file mode 100644 index 0000000..d50dfd6 --- /dev/null +++ b/crates/spatialrust-io/src/manifest.rs @@ -0,0 +1,176 @@ +//! Checksummed file receipts and dataset manifests. + +use std::fs::File; +use std::io::{BufReader, Read}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::IoError; + +/// Current JSON schema version for [`DatasetManifest`]. +pub const DATASET_MANIFEST_VERSION: u32 = 1; + +/// Logical role of a file in a dataset operation. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ReceiptRole { + /// File consumed by an operation. + Input, + /// File produced by an operation. + Output, + /// File associated with an operation but not consumed or produced by it. + Auxiliary, +} + +/// Size and SHA-256 receipt for one local file or URI source. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct FileReceipt { + /// Role of the source in the operation. + pub role: ReceiptRole, + /// Resolved local path or URI written as a path-like JSON string. + pub path: PathBuf, + /// Number of bytes observed while hashing a local file. + #[serde(skip_serializing_if = "Option::is_none")] + pub size_bytes: Option, + /// Lowercase hexadecimal SHA-256 digest for a local file. + #[serde(skip_serializing_if = "Option::is_none")] + pub sha256: Option, +} + +impl FileReceipt { + /// Hashes a local file and returns its size/checksum receipt. + pub fn from_path(role: ReceiptRole, path: impl AsRef) -> Result { + let path = path.as_ref(); + let file = File::open(path)?; + let mut reader = BufReader::new(file); + let mut hasher = Sha256::new(); + let mut bytes = 0_u64; + let mut buffer = [0_u8; 64 * 1024]; + + loop { + let read = reader.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + bytes = bytes.checked_add(read as u64).ok_or_else(|| { + IoError::Manifest(format!("file size overflow while hashing `{}`", path.display())) + })?; + } + + let digest = hasher.finalize(); + Ok(Self { + role, + path: path.to_path_buf(), + size_bytes: Some(bytes), + sha256: Some(hex_digest(&digest)), + }) + } + + /// Records a URI source whose bytes were not materialized locally. + #[must_use] + pub fn from_uri(role: ReceiptRole, uri: impl Into) -> Self { + Self { role, path: uri.into(), size_bytes: None, sha256: None } + } +} + +/// JSON manifest containing the files associated with one dataset operation. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct DatasetManifest { + /// Version of the manifest JSON schema. + pub version: u32, + /// File and URI receipts in operation order. + pub entries: Vec, +} + +impl Default for DatasetManifest { + fn default() -> Self { + Self::new() + } +} + +impl DatasetManifest { + /// Creates an empty manifest at the current schema version. + #[must_use] + pub const fn new() -> Self { + Self { version: DATASET_MANIFEST_VERSION, entries: Vec::new() } + } + + /// Adds a checksummed local file receipt. + pub fn add_file(&mut self, role: ReceiptRole, path: impl AsRef) -> Result<(), IoError> { + self.entries.push(FileReceipt::from_path(role, path)?); + Ok(()) + } + + /// Adds a URI receipt without claiming a local byte count or checksum. + pub fn add_uri(&mut self, role: ReceiptRole, uri: impl Into) { + self.entries.push(FileReceipt::from_uri(role, uri)); + } + + /// Serializes this manifest as pretty-printed JSON. + pub fn to_json(&self) -> Result { + serde_json::to_string_pretty(self).map_err(|error| { + IoError::Manifest(format!("cannot serialize dataset manifest: {error}")) + }) + } + + /// Writes this manifest as JSON, creating its parent directory if needed. + pub fn write_json(&self, path: impl AsRef) -> Result<(), IoError> { + let path = path.as_ref(); + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent)?; + } + } + std::fs::write(path, format!("{}\n", self.to_json()?))?; + Ok(()) + } +} + +fn hex_digest(bytes: &[u8]) -> String { + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + output.push_str(&format!("{byte:02x}")); + } + output +} + +#[cfg(test)] +mod tests { + use super::{DatasetManifest, FileReceipt, ReceiptRole, DATASET_MANIFEST_VERSION}; + + #[test] + fn hashes_file_and_records_size() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("scan.bin"); + std::fs::write(&path, b"hello").unwrap(); + + let receipt = FileReceipt::from_path(ReceiptRole::Input, &path).unwrap(); + assert_eq!(receipt.size_bytes, Some(5)); + assert_eq!( + receipt.sha256.as_deref(), + Some("2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824") + ); + } + + #[test] + fn serializes_local_and_uri_entries() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("scan.bin"); + std::fs::write(&path, b"data").unwrap(); + + let mut manifest = DatasetManifest::new(); + manifest.add_file(ReceiptRole::Input, &path).unwrap(); + manifest.add_uri(ReceiptRole::Auxiliary, "https://example.test/scan.copc.laz"); + let json = manifest.to_json().unwrap(); + let decoded: DatasetManifest = serde_json::from_str(&json).unwrap(); + + assert_eq!(decoded.version, DATASET_MANIFEST_VERSION); + assert_eq!(decoded.entries.len(), 2); + assert_eq!(decoded.entries[0].role, ReceiptRole::Input); + assert_eq!(decoded.entries[0].size_bytes, Some(4)); + assert_eq!(decoded.entries[1].sha256, None); + } +} diff --git a/crates/spatialrust-io/src/storage.rs b/crates/spatialrust-io/src/storage.rs new file mode 100644 index 0000000..e683db6 --- /dev/null +++ b/crates/spatialrust-io/src/storage.rs @@ -0,0 +1,111 @@ +//! Explicit input/output roots for local spatial data. + +use std::path::{Component, Path, PathBuf}; + +use crate::IoError; + +/// Optional roots used to resolve logical input and output paths. +/// +/// Relative paths are joined to their corresponding root. Absolute paths are +/// always treated as explicit locations and bypass the roots. Relative paths +/// containing `..` are rejected so a logical dataset path cannot escape its +/// configured root. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StorageRoots { + input_root: Option, + output_root: Option, +} + +impl StorageRoots { + /// Creates a root mapping. Either root may be omitted. + #[must_use] + pub const fn new(input_root: Option, output_root: Option) -> Self { + Self { input_root, output_root } + } + + /// Returns the configured input root, if any. + #[must_use] + pub fn input_root(&self) -> Option<&Path> { + self.input_root.as_deref() + } + + /// Returns the configured output root, if any. + #[must_use] + pub fn output_root(&self) -> Option<&Path> { + self.output_root.as_deref() + } + + /// Resolves a logical input path against the configured input root. + pub fn resolve_input(&self, path: impl AsRef) -> Result { + resolve_path(self.input_root.as_deref(), path.as_ref(), "input") + } + + /// Resolves a logical output path against the configured output root. + pub fn resolve_output(&self, path: impl AsRef) -> Result { + resolve_path(self.output_root.as_deref(), path.as_ref(), "output") + } + + /// Creates the parent directory for an explicit output path when needed. + pub fn ensure_output_parent(&self, path: impl AsRef) -> Result<(), IoError> { + if let Some(parent) = path.as_ref().parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent)?; + } + } + Ok(()) + } +} + +fn resolve_path(root: Option<&Path>, path: &Path, kind: &str) -> Result { + if path.is_absolute() || root.is_none() { + return Ok(path.to_path_buf()); + } + + if path.components().any(|component| component == Component::ParentDir) { + return Err(IoError::Storage(format!( + "relative {kind} path `{}` must not contain `..`", + path.display() + ))); + } + + Ok(root.expect("root checked above").join(path)) +} + +#[cfg(test)] +mod tests { + use super::StorageRoots; + use std::path::PathBuf; + + #[test] + fn resolves_relative_paths_per_direction() { + let roots = StorageRoots::new( + Some(PathBuf::from("/mnt/input")), + Some(PathBuf::from("/mnt/output")), + ); + assert_eq!(roots.resolve_input("scan.las").unwrap(), PathBuf::from("/mnt/input/scan.las")); + assert_eq!( + roots.resolve_output("runs/result.las").unwrap(), + PathBuf::from("/mnt/output/runs/result.las") + ); + } + + #[test] + fn absolute_paths_bypass_roots() { + let roots = StorageRoots::new( + Some(PathBuf::from("/mnt/input")), + Some(PathBuf::from("/mnt/output")), + ); + assert_eq!(roots.resolve_input("/tmp/scan.las").unwrap(), PathBuf::from("/tmp/scan.las")); + assert_eq!( + roots.resolve_output("/tmp/result.las").unwrap(), + PathBuf::from("/tmp/result.las") + ); + } + + #[test] + fn rejects_relative_root_escape() { + let roots = StorageRoots::new(Some(PathBuf::from("/mnt/input")), None); + let error = roots.resolve_input("../private/scan.las").unwrap_err(); + assert!(error.to_string().contains("must not contain `..`")); + } +} diff --git a/crates/spatialrust-py/README.md b/crates/spatialrust-py/README.md index 6139e45..a3e6c92 100644 --- a/crates/spatialrust-py/README.md +++ b/crates/spatialrust-py/README.md @@ -113,6 +113,12 @@ print(stream.receipt_json()) Call `stream.cancel()` to stop cooperatively at a chunk boundary. Retained Python chunks are caller-owned and are outside the native pipeline budget. +Pass `input_root="/media/sasaki/aiueo/datasets"` to resolve a relative stream +path on an external SSD. `read()` and `write()` accept `input_root`/ +`output_root`; `write(..., manifest_path=...)` records the output size and +SHA-256. Use `run_pipeline_files()` when one manifest should contain both the +input and output receipts. + ## Quickstart ```python @@ -148,7 +154,7 @@ reloaded = sr.read("labeled.las") | `PointCloud.xyz()` | XYZ as an `(N, 3)` float32 array | | `PointCloud.labels()` | Cluster labels as `(N,)` int32, or `None` | | `PointCloud.field_names()` / `len(cloud)` | Schema fields / point count | -| `read(path)` / `write(path, cloud)` | IO by file extension | +| `read(path, input_root=None)` / `write(path, cloud, output_root=None, manifest_path=None)` | Explicit external roots and output receipt | | `voxel_downsample(cloud, leaf_size, policy="auto")` | Voxel-grid downsample | | `crop_box(cloud, min, max, invert=False)` | Keep/drop points inside an AABB | | `pass_through(cloud, field, min, max, invert=False)` | Keep/drop points by a field's value range | @@ -167,6 +173,7 @@ reloaded = sr.read("labeled.las") | `statistical_outlier_removal(cloud, k_neighbors=16, std_mul=1.0)` | Drop points far from their k-NN (SOR) | | `radius_outlier_removal(cloud, radius=0.5, min_neighbors=4)` | Drop points with too few neighbors in radius (ROR) | | `run_pipeline(cloud, leaf_size=0.05, cluster_tolerance=None, min_cluster_size=None, plane_distance=None, policy="auto")` | Full MVP pipeline with resolved backend and transfer-byte properties | +| `run_pipeline_files(input, output, input_root=None, output_root=None, manifest_path=None, ...)` | File-backed MVP pipeline with explicit roots and input/output manifest | | `iss_keypoints(cloud, salient_radius=0.2, non_max_radius=0.15, ...)` | ISS keypoints (sparse salient sub-cloud) | | `orient_normals(cloud, k_neighbors=15)` | Estimate normals, then orient them consistently (MST) | | `detect_boundary(cloud, search_radius=0.1, angle_threshold=1.5708, ...)` | Boundary / edge points (sparse sub-cloud) | diff --git a/crates/spatialrust-py/spatialrust.pyi b/crates/spatialrust-py/spatialrust.pyi index 56966dc..b3cb766 100644 --- a/crates/spatialrust-py/spatialrust.pyi +++ b/crates/spatialrust-py/spatialrust.pyi @@ -25,7 +25,7 @@ __all__: list[str] = [ "open_point_cloud_stream", "write", "voxel_downsample", "crop_box", "pass_through", "iss_keypoints", "orient_normals", "detect_boundary", "mls_smooth", "farthest_point_sampling", - "statistical_outlier_removal", "radius_outlier_removal", "run_pipeline", + "statistical_outlier_removal", "radius_outlier_removal", "run_pipeline", "run_pipeline_files", "region_growing", "dbscan", "ground_segmentation", "segment_multi_plane", "ransac_sphere", "ransac_cylinder", "chamfer_distance", "hausdorff_distance", "apply_transform", "recenter", "scale", "normalize_unit_sphere", "merge", @@ -604,8 +604,13 @@ class RegistrationResult: # --------------------------------------------------------------------------- # # IO # --------------------------------------------------------------------------- # -def read(path: str) -> PointCloud: ... -def write(path: str, cloud: PointCloud) -> None: ... +def read(path: str, input_root: Optional[str] = ...) -> PointCloud: ... +def write( + path: str, + cloud: PointCloud, + output_root: Optional[str] = ..., + manifest_path: Optional[str] = ..., +) -> None: ... def open_point_cloud_stream( path: str, chunk_points: int = ..., @@ -617,6 +622,7 @@ def open_point_cloud_stream( max_runs: int = ..., spool_dir: Optional[str] = ..., spool_limit_bytes: int = ..., + input_root: Optional[str] = ..., ) -> PointCloudStream: ... # --------------------------------------------------------------------------- # @@ -761,6 +767,18 @@ def run_pipeline( plane_distance: Optional[float] = ..., policy: str = ..., ) -> PipelineResult: ... +def run_pipeline_files( + input: str, + output: str, + input_root: Optional[str] = ..., + output_root: Optional[str] = ..., + manifest_path: Optional[str] = ..., + leaf_size: float = ..., + cluster_tolerance: Optional[float] = ..., + min_cluster_size: Optional[int] = ..., + plane_distance: Optional[float] = ..., + policy: str = ..., +) -> PipelineResult: ... def register_icp( source: PointCloud, target: PointCloud, diff --git a/crates/spatialrust-py/src/lib.rs b/crates/spatialrust-py/src/lib.rs index e2f2ac1..c16a8fb 100644 --- a/crates/spatialrust-py/src/lib.rs +++ b/crates/spatialrust-py/src/lib.rs @@ -8,6 +8,8 @@ #![allow(clippy::useless_conversion)] #![deny(unsafe_code)] +use std::path::PathBuf; + #[allow(unsafe_code)] mod dlpack_capsule; mod viewer; @@ -50,6 +52,7 @@ use spatialrust::image_io::{ decode_path as decode_image_path, encode_path as encode_image_path, DecodeOptions, DecodedMetadata, DecodedPixels, EncodeOptions, ImageFileFormat, }; +use spatialrust::io::{DatasetManifest, ReceiptRole, StorageRoots}; use spatialrust::math::{Mat3, Mat4, Vec2, Vec3}; use spatialrust::metrics::{chamfer_distance as chamfer, hausdorff_distance as hausdorff}; use spatialrust::pipeline::{ @@ -1091,7 +1094,8 @@ impl PyPointCloudStream { run_points=65536, max_runs=1024, spool_dir=None, - spool_limit_bytes=2147483648 + spool_limit_bytes=2147483648, + input_root=None ))] #[allow(clippy::too_many_arguments)] fn open_point_cloud_stream( @@ -1105,6 +1109,7 @@ fn open_point_cloud_stream( max_runs: usize, spool_dir: Option, spool_limit_bytes: u64, + input_root: Option, ) -> PyResult { let options = StreamOptions::new( chunk_points, @@ -1112,6 +1117,8 @@ fn open_point_cloud_stream( ) .map_err(to_py_err)?; let cancellation = CancellationToken::default(); + let roots = StorageRoots::new(input_root.map(PathBuf::from), None); + let path = roots.resolve_input(&path).map_err(to_py_err)?.to_string_lossy().into_owned(); let source = open_python_stream_source(&path, options, cancellation.clone()).map_err(to_py_err)?; let mut pipeline = StreamingPipeline::new(source, path.clone()).map_err(to_py_err)?; @@ -1533,15 +1540,34 @@ fn hausdorff_distance(a: &PyPointCloud, b: &PyPointCloud) -> PyResult { /// Reads a point cloud from a file (PCD/PLY/LAS/COPC by extension). #[pyfunction] -fn read(path: &str) -> PyResult { - let inner = read_point_cloud_file(path).map_err(to_py_err)?; +#[pyo3(signature = (path, input_root=None))] +fn read(path: &str, input_root: Option) -> PyResult { + let roots = StorageRoots::new(input_root.map(PathBuf::from), None); + let resolved = roots.resolve_input(path).map_err(to_py_err)?; + let inner = read_point_cloud_file(&resolved).map_err(to_py_err)?; Ok(PyPointCloud { inner }) } /// Writes a point cloud to a file (format chosen by extension). #[pyfunction] -fn write(path: &str, cloud: &PyPointCloud) -> PyResult<()> { - write_point_cloud_file(path, &cloud.inner).map_err(to_py_err) +#[pyo3(signature = (path, cloud, output_root=None, manifest_path=None))] +fn write( + path: &str, + cloud: &PyPointCloud, + output_root: Option, + manifest_path: Option, +) -> PyResult<()> { + let roots = StorageRoots::new(None, output_root.map(PathBuf::from)); + let resolved = roots.resolve_output(path).map_err(to_py_err)?; + roots.ensure_output_parent(&resolved).map_err(to_py_err)?; + write_point_cloud_file(&resolved, &cloud.inner).map_err(to_py_err)?; + if let Some(manifest_path) = manifest_path { + let manifest_path = roots.resolve_output(manifest_path).map_err(to_py_err)?; + let mut manifest = DatasetManifest::new(); + manifest.add_file(ReceiptRole::Output, &resolved).map_err(to_py_err)?; + manifest.write_json(manifest_path).map_err(to_py_err)?; + } + Ok(()) } /// Voxel-grid downsamples a cloud. `policy` is one of "auto", "cpu", "cpu-single". @@ -1751,6 +1777,93 @@ fn run_pipeline( }) } +/// Reads, runs the MVP pipeline, and writes a point cloud using explicit data roots. +/// +/// Relative `input` and `output` paths are resolved under `input_root` and +/// `output_root`. When `manifest_path` is supplied, the manifest contains +/// local input/output size and SHA-256 receipts. +#[pyfunction] +#[pyo3(signature = ( + input, + output, + input_root=None, + output_root=None, + manifest_path=None, + leaf_size=0.05, + cluster_tolerance=None, + min_cluster_size=None, + plane_distance=None, + policy="auto" +))] +#[allow(clippy::too_many_arguments)] +fn run_pipeline_files( + input: &str, + output: &str, + input_root: Option, + output_root: Option, + manifest_path: Option, + leaf_size: f32, + cluster_tolerance: Option, + min_cluster_size: Option, + plane_distance: Option, + policy: &str, +) -> PyResult { + let roots = StorageRoots::new( + input_root.map(PathBuf::from), + output_root.map(PathBuf::from), + ); + let input_path = roots.resolve_input(input).map_err(to_py_err)?; + let output_path = roots.resolve_output(output).map_err(to_py_err)?; + let manifest_path = manifest_path + .map(|path| roots.resolve_output(path)) + .transpose() + .map_err(to_py_err)?; + let cloud = read_point_cloud_file(&input_path).map_err(to_py_err)?; + + let mut config = MvpPipelineConfig::with_voxel_leaf_size(leaf_size); + config.voxel_policy = parse_policy(policy)?; + if let Some(tol) = cluster_tolerance { + config.cluster.cluster_tolerance = tol; + } + if let Some(min) = min_cluster_size { + config.cluster.min_cluster_size = min; + } + if let Some(dist) = plane_distance { + config.plane.distance_threshold = dist; + } + let result = MvpPipeline::new(config).run(&cloud).map_err(to_py_err)?; + + roots.ensure_output_parent(&output_path).map_err(to_py_err)?; + write_point_cloud_file(&output_path, &result.output).map_err(to_py_err)?; + if let Some(manifest_path) = manifest_path { + let mut manifest = DatasetManifest::new(); + manifest.add_file(ReceiptRole::Input, &input_path).map_err(to_py_err)?; + manifest.add_file(ReceiptRole::Output, &output_path).map_err(to_py_err)?; + manifest.write_json(manifest_path).map_err(to_py_err)?; + } + + let normal = result.plane.model.normal; + let transfers = result.receipt.transfer_stats(); + let resolved_policies = vec![ + format!("{:?}", result.receipt.voxel.resolved_policy()), + format!("{:?}", result.receipt.normals.resolved_policy()), + format!("{:?}", result.receipt.plane.resolved_policy()), + format!("{:?}", result.receipt.clusters.resolved_policy()), + ]; + Ok(PyPipelineResult { + output: PyPointCloud { inner: result.output }, + downsampled: PyPointCloud { inner: result.downsampled }, + cluster_count: result.clusters.cluster_count, + cluster_sizes: result.clusters.cluster_sizes, + plane_inliers: result.plane.inlier_count, + plane_normal: (normal.x, normal.y, normal.z), + resolved_policies, + host_to_device_bytes: transfers.host_to_device_bytes(), + device_to_device_bytes: transfers.device_to_device_bytes(), + device_to_host_bytes: transfers.device_to_host_bytes(), + }) +} + /// Normal-based region growing: estimates normals, then grows smooth regions. /// /// `smoothness_deg` is the maximum angle (degrees) between neighboring normals @@ -4509,6 +4622,7 @@ fn spatialrust_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(statistical_outlier_removal, m)?)?; m.add_function(wrap_pyfunction!(radius_outlier_removal, m)?)?; m.add_function(wrap_pyfunction!(run_pipeline, m)?)?; + m.add_function(wrap_pyfunction!(run_pipeline_files, m)?)?; m.add_function(wrap_pyfunction!(region_growing, m)?)?; m.add_function(wrap_pyfunction!(dbscan, m)?)?; m.add_function(wrap_pyfunction!(ground_segmentation, m)?)?; diff --git a/crates/spatialrust-py/tests/test_bindings.py b/crates/spatialrust-py/tests/test_bindings.py index 5636d11..8ec5919 100644 --- a/crates/spatialrust-py/tests/test_bindings.py +++ b/crates/spatialrust-py/tests/test_bindings.py @@ -137,6 +137,7 @@ def test_exports_present(): for name in ( "PointCloud", "PointCloudStream", "open_point_cloud_stream", "voxel_downsample", "dbscan", "register_icp", + "run_pipeline_files", "voxelize", "knn_graph", "chamfer_distance", "oriented_bounding_box", "rgbd_to_point_cloud", "depth_to_xyz", "resize_image", "letterbox_image", "normalize_image_chw", "resize_normalize_image_chw", @@ -169,6 +170,37 @@ def test_bounded_point_cloud_stream_and_receipt(tmp_path): assert receipt["output_points"] == 2 +def test_external_storage_roots_and_manifest(tmp_path): + input_root = tmp_path / "input" + output_root = tmp_path / "output" + input_root.mkdir() + (input_root / "scan.pcd").write_text( + "VERSION .7\nFIELDS x y z\nSIZE 4 4 4\nTYPE F F F\n" + "COUNT 1 1 1\nWIDTH 3\nHEIGHT 1\nPOINTS 3\nDATA ascii\n" + "0 0 0\n1 0 0\n2 0 0\n", + encoding="ascii", + ) + + cloud = sr.read("scan.pcd", input_root=str(input_root)) + sr.write( + "runs/copy.pcd", + cloud, + output_root=str(output_root), + manifest_path="receipts/copy.json", + ) + + manifest_path = output_root / "receipts" / "copy.json" + assert (output_root / "runs" / "copy.pcd").exists() + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert manifest["version"] == 1 + assert manifest["entries"][0]["role"] == "output" + assert manifest["entries"][0]["size_bytes"] > 0 + assert len(manifest["entries"][0]["sha256"]) == 64 + + with pytest.raises(ValueError): + sr.read("../scan.pcd", input_root=str(input_root)) + + # --------------------------------------------------------------------------- # # PointCloud round-trips # --------------------------------------------------------------------------- # diff --git a/crates/spatialrust/Cargo.toml b/crates/spatialrust/Cargo.toml index 056fa01..9c463e2 100644 --- a/crates/spatialrust/Cargo.toml +++ b/crates/spatialrust/Cargo.toml @@ -23,6 +23,7 @@ io-laz = ["spatialrust-io/io-laz"] io-e57 = ["spatialrust-io/io-e57"] io-copc = ["spatialrust-io/io-copc"] io-copc-http = ["io-copc", "spatialrust-io/io-copc-http"] +io-manifest = ["spatialrust-io/io-manifest"] io-streaming = ["records", "spatialrust-io/streaming"] search-kdtree = ["spatialrust-search/search-kdtree"] search-parallel = ["search-kdtree", "spatialrust-search/parallel"] @@ -115,6 +116,7 @@ mvp = [ "segment-euclidean", "register-icp", "pipeline-mvp", + "io-manifest", ] mvp-http = ["mvp", "io-copc-http"] pipeline-mvp = ["spatialrust-pipeline/pipeline-mvp"] @@ -131,6 +133,7 @@ streaming-cli = [ "io-laz", "io-copc-http", "records-receipt-json", + "io-manifest", "dep:ctrlc", ] pipeline-mvp-gpu = [ diff --git a/crates/spatialrust/src/bin/spatialrust_mvp.rs b/crates/spatialrust/src/bin/spatialrust_mvp.rs index 9424e99..6fa99eb 100644 --- a/crates/spatialrust/src/bin/spatialrust_mvp.rs +++ b/crates/spatialrust/src/bin/spatialrust_mvp.rs @@ -10,13 +10,18 @@ //! --voxel-mode approximate --leaf-size 0.2 scan.las out.las //! ``` -use std::{env, path::Path, process::ExitCode, time::Instant}; +use std::{ + env, + path::{Path, PathBuf}, + process::ExitCode, + time::Instant, +}; use spatialrust::{ detect_point_cloud_format, read_copc_file_info, read_copc_file_with_query, - read_point_cloud_file, write_point_cloud_file, CopcBounds, CopcQuery, ExecutionPolicy, - MvpPipeline, MvpPipelineConfig, PointCloudFileFormat, VoxelAggregationMode, - VoxelGridDownsampleConfig, + read_point_cloud_file, write_point_cloud_file, CopcBounds, CopcQuery, DatasetManifest, + ExecutionPolicy, MvpPipeline, MvpPipelineConfig, PointCloudFileFormat, ReceiptRole, + StorageRoots, VoxelAggregationMode, VoxelGridDownsampleConfig, }; #[cfg(feature = "io-copc-http")] use spatialrust::{read_copc_url_info, read_copc_url_with_query}; @@ -52,6 +57,9 @@ Options: --resolution COPC max point spacing LOD (requires COPC input; uses root bounds when --bounds is omitted) --repeat Run the MVP pipeline N times (default: 1); logs per-iteration timing + --input-root Resolve relative INPUT paths under this root (external SSD friendly) + --output-root Resolve relative OUTPUT/manifest paths under this root + --manifest Write input/output size and SHA-256 receipts as JSON -h, --help Show this help " ); @@ -64,9 +72,7 @@ fn parse_execution_policy(value: &str, stage: &str) -> Result { #[cfg(not(feature = "pipeline-mvp-gpu"))] { - return Err(format!( - "GPU {stage} policy requires `--features mvp,pipeline-mvp-gpu`" - )); + Err(format!("GPU {stage} policy requires `--features mvp,pipeline-mvp-gpu`")) } #[cfg(feature = "pipeline-mvp-gpu")] { @@ -302,6 +308,9 @@ fn run() -> Result<(), Box> { let mut cluster_policy = ExecutionPolicy::Auto; let mut copc = CopcQueryOptions::default(); let mut repeat = 1_usize; + let mut input_root = None; + let mut output_root = None; + let mut manifest_path = None; let mut input_path = None; let mut output_path = None; @@ -347,6 +356,18 @@ fn run() -> Result<(), Box> { let value = args.next().ok_or("--repeat requires a positive integer")?; repeat = parse_repeat(&value)?; } + "--input-root" => { + input_root = + Some(PathBuf::from(args.next().ok_or("--input-root requires a path")?)); + } + "--output-root" => { + output_root = + Some(PathBuf::from(args.next().ok_or("--output-root requires a path")?)); + } + "--manifest" => { + manifest_path = + Some(PathBuf::from(args.next().ok_or("--manifest requires a path")?)); + } value if value.starts_with('-') => { return Err(format!("unknown option `{value}`").into()); } @@ -362,15 +383,24 @@ fn run() -> Result<(), Box> { } } - let input_path = input_path.ok_or("missing INPUT path")?; - let output_path = output_path.ok_or("missing OUTPUT path")?; + let input_arg = input_path.ok_or("missing INPUT path")?; + let output_arg = output_path.ok_or("missing OUTPUT path")?; + let roots = StorageRoots::new(input_root, output_root); + let input_path = if is_http_copc_input(&input_arg) { + PathBuf::from(&input_arg) + } else { + roots.resolve_input(&input_arg)? + }; + let output_path = roots.resolve_output(&output_arg)?; + let manifest_path = manifest_path.map(|path| roots.resolve_output(path)).transpose()?; + let input_path_text = input_path.to_string_lossy().into_owned(); - if !is_http_copc_input(&input_path) && !Path::new(&input_path).exists() { - return Err(format!("input file not found: {input_path}").into()); + if !is_http_copc_input(&input_path_text) && !Path::new(&input_path).exists() { + return Err(format!("input file not found: {}", input_path.display()).into()); } - eprintln!("loading {input_path}"); - let input = load_input(&input_path, copc)?; + eprintln!("loading {}", input_path.display()); + let input = load_input(&input_path_text, copc)?; let input_points = input.len(); eprintln!("input points: {input_points}"); @@ -406,8 +436,21 @@ fn run() -> Result<(), Box> { let result = result.expect("pipeline produced no result"); let elapsed = *timings.last().expect("repeat timings"); + roots.ensure_output_parent(&output_path)?; write_point_cloud_file(&output_path, &result.output)?; + if let Some(manifest_path) = manifest_path { + let mut manifest = DatasetManifest::new(); + if is_http_copc_input(&input_path_text) { + manifest.add_uri(ReceiptRole::Input, input_path_text.clone()); + } else { + manifest.add_file(ReceiptRole::Input, &input_path)?; + } + manifest.add_file(ReceiptRole::Output, &output_path)?; + manifest.write_json(&manifest_path)?; + eprintln!("wrote manifest {}", manifest_path.display()); + } + eprintln!("output points: {}", result.output.len()); eprintln!("plane inliers: {}", result.plane.inlier_count); eprintln!("clusters: {}", result.clusters.cluster_count); @@ -426,7 +469,7 @@ fn run() -> Result<(), Box> { transfers.device_to_host_bytes(), ); eprintln!("elapsed: {:.3?}", elapsed); - eprintln!("wrote {output_path}"); + eprintln!("wrote {}", output_path.display()); Ok(()) } diff --git a/crates/spatialrust/src/bin/spatialrust_stream.rs b/crates/spatialrust/src/bin/spatialrust_stream.rs index b952a13..1d3502a 100644 --- a/crates/spatialrust/src/bin/spatialrust_stream.rs +++ b/crates/spatialrust/src/bin/spatialrust_stream.rs @@ -4,8 +4,8 @@ use std::error::Error; use std::path::{Path, PathBuf}; use spatialrust::io::{ - CopcChunkSource, LasChunkSink, LasChunkSource, LasWriteFormat, PcdChunkSource, PlyChunkSource, - SpoolOptions, + CopcChunkSource, DatasetManifest, LasChunkSink, LasChunkSource, LasWriteFormat, PcdChunkSource, + PlyChunkSource, ReceiptRole, SpoolOptions, StorageRoots, }; use spatialrust::math::{Mat3, Mat4, Vec3}; use spatialrust::pipeline::{StreamingPipeline, StreamingVoxelConfig}; @@ -28,6 +28,9 @@ struct Config { spool_dir: PathBuf, spool_bytes: u64, receipt: Option, + manifest: Option, + input_root: Option, + output_root: Option, } fn main() { @@ -39,13 +42,24 @@ fn main() { fn run() -> Result<(), Box> { let config = parse_args(std::env::args().skip(1))?; + let roots = StorageRoots::new(config.input_root.clone(), config.output_root.clone()); + let input_path = if config.input.starts_with("http://") || config.input.starts_with("https://") + { + PathBuf::from(&config.input) + } else { + roots.resolve_input(&config.input)? + }; + let output_path = roots.resolve_output(&config.output)?; + let receipt_path = config.receipt.map(|path| roots.resolve_output(path)).transpose()?; + let manifest_path = config.manifest.map(|path| roots.resolve_output(path)).transpose()?; + let input_text = input_path.to_string_lossy().into_owned(); let options = StreamOptions::new(config.chunk_points, MemoryBudget::new(config.memory_bytes)?)?; let cancellation = CancellationToken::default(); let interrupt = cancellation.clone(); ctrlc::set_handler(move || interrupt.cancel())?; - let source = open_source(&config.input, options, cancellation)?; - let mut pipeline = StreamingPipeline::new(source, config.input.clone())?; + let source = open_source(&input_text, options, cancellation)?; + let mut pipeline = StreamingPipeline::new(source, input_text.clone())?; if let Some((min, max)) = config.crop { pipeline = pipeline.crop(min, max, false)?; } @@ -65,16 +79,29 @@ fn run() -> Result<(), Box> { )?)?; } - let format = output_format(&config.output)?; + roots.ensure_output_parent(&output_path)?; + let format = output_format(&output_path)?; let mut sink = - LasChunkSink::create_open_ended(&config.output, pipeline.schema().clone(), format)?; + LasChunkSink::create_open_ended(&output_path, pipeline.schema().clone(), format)?; let receipt = pipeline.run_to_sink(&mut sink)?; let json = receipt.to_json()?; - if let Some(path) = config.receipt { + if let Some(path) = receipt_path { + roots.ensure_output_parent(&path)?; std::fs::write(path, format!("{json}\n"))?; } else { println!("{json}"); } + if let Some(path) = manifest_path { + let mut manifest = DatasetManifest::new(); + if input_text.starts_with("http://") || input_text.starts_with("https://") { + manifest.add_uri(ReceiptRole::Input, input_text); + } else { + manifest.add_file(ReceiptRole::Input, &input_path)?; + } + manifest.add_file(ReceiptRole::Output, &output_path)?; + manifest.write_json(&path)?; + eprintln!("wrote manifest {}", path.display()); + } Ok(()) } @@ -134,6 +161,9 @@ fn parse_args(args: impl IntoIterator) -> Result) -> Result config.spool_bytes = parse_one(&mut args, &flag)?, "--spool-dir" => config.spool_dir = PathBuf::from(next_value(&mut args, &flag)?), "--receipt" => config.receipt = Some(PathBuf::from(next_value(&mut args, &flag)?)), + "--manifest" => config.manifest = Some(PathBuf::from(next_value(&mut args, &flag)?)), + "--input-root" => { + config.input_root = Some(PathBuf::from(next_value(&mut args, &flag)?)); + } + "--output-root" => { + config.output_root = Some(PathBuf::from(next_value(&mut args, &flag)?)); + } "--translate" => { config.translation = Some([ parse_one(&mut args, &flag)?, @@ -191,7 +228,7 @@ fn usage() -> String { "usage: spatialrust-stream INPUT OUTPUT [--chunk-points N] [--memory-budget BYTES] \ [--crop MINX MINY MINZ MAXX MAXY MAXZ] [--translate X Y Z] [--voxel LEAF] \ [--run-points N] [--max-runs N] [--spool-dir DIR] [--spool-limit BYTES] \ - [--receipt PATH]" + [--receipt PATH] [--manifest PATH] [--input-root DIR] [--output-root DIR]" .into() } @@ -214,4 +251,32 @@ mod tests { assert_eq!(config.crop, Some(([0.0, 1.0, 2.0], [3.0, 4.0, 5.0]))); assert_eq!(output_format(&config.output).unwrap(), LasWriteFormat::Laz); } + + #[test] + fn parses_external_storage_roots_and_manifest() { + let config = parse_args( + [ + "scan.pcd", + "runs/out.laz", + "--input-root", + "/media/sasaki/aiueo/input", + "--output-root", + "/media/sasaki/aiueo/output", + "--manifest", + "runs/out.json", + ] + .into_iter() + .map(str::to_owned), + ) + .unwrap(); + assert_eq!( + config.input_root.as_deref(), + Some(std::path::Path::new("/media/sasaki/aiueo/input")) + ); + assert_eq!( + config.output_root.as_deref(), + Some(std::path::Path::new("/media/sasaki/aiueo/output")) + ); + assert_eq!(config.manifest.as_deref(), Some(std::path::Path::new("runs/out.json"))); + } } diff --git a/crates/spatialrust/src/lib.rs b/crates/spatialrust/src/lib.rs index 2ad6fd8..17b1982 100644 --- a/crates/spatialrust/src/lib.rs +++ b/crates/spatialrust/src/lib.rs @@ -108,10 +108,13 @@ pub use spatialrust_io::{ read_copc_url, read_copc_url_info, read_copc_url_with_query, HttpByteSource, }; +pub use spatialrust_io::StorageRoots; pub use spatialrust_io::{ detect_point_cloud_format, read_point_cloud_file, read_point_cloud_file_with_format, write_point_cloud_file, write_point_cloud_file_with_format, PointCloudFileFormat, }; +#[cfg(feature = "io-manifest")] +pub use spatialrust_io::{DatasetManifest, FileReceipt, ReceiptRole, DATASET_MANIFEST_VERSION}; #[cfg(feature = "search-kdtree")] pub use spatialrust_search::{ diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f4de8be..93e0ae2 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -42,6 +42,9 @@ North star: **Rust-native spatial intelligence: capture, understand, reconstruct Post-MVP additions: - Unified file IO via `read_point_cloud_file` / `write_point_cloud_file` +- Explicit `spatialrust_io::StorageRoots` for external input/output storage; + `io-manifest` adds checksummed size receipts without adding storage policy to + `spatialrust-core` - `MvpPipelineConfig::*_policy` for feature-gated GPU MVP stages - `MvpPipelineResult::receipt` for per-stage backend and transfer accounting diff --git a/docs/EXTERNAL_STORAGE.md b/docs/EXTERNAL_STORAGE.md new file mode 100644 index 0000000..7a0518c --- /dev/null +++ b/docs/EXTERNAL_STORAGE.md @@ -0,0 +1,67 @@ +# External storage for point-cloud IO + +SpatialRust keeps data placement explicit. The point-cloud core does not know +about disks, and IO readers/writers do not silently copy a dataset to another +device. Applications can provide separate input and output roots at the CLI or +Python boundary. + +## Rust and CLI + +Relative logical paths are resolved under the matching root. Absolute paths +remain explicit and bypass the root; relative paths containing `..` are +rejected. + +```bash +cargo run -p spatialrust --features mvp --bin spatialrust-mvp -- \ + --input-root /media/sasaki/aiueo/datasets \ + --output-root /media/sasaki/aiueo/spatialrust-results \ + --manifest runs/scan.json \ + boreas/scan.las runs/scan.ply +``` + +The bounded streaming CLI accepts the same root flags and writes the existing +workflow receipt separately from the file manifest: + +```bash +cargo run -p spatialrust --features streaming-cli --bin spatialrust-stream -- \ + --input-root /media/sasaki/aiueo/datasets \ + --output-root /media/sasaki/aiueo/spatialrust-results \ + --receipt runs/stream-receipt.json \ + --manifest runs/stream-manifest.json \ + boreas/scan.pcd runs/scan.laz +``` + +The `spatialrust-io/io-manifest` feature adds `DatasetManifest`, +`FileReceipt`, and SHA-256 hashing. A manifest has version `1` and ordered +`entries`; local input/output entries contain `size_bytes` and `sha256`. An +HTTP(S) COPC input is recorded as an `input` URI entry without a local size or +checksum because the CLI does not materialize it before streaming. + +## Python + +The binding exposes the same explicit roots without changing the default +behavior: + +```python +import spatialrust as sr + +cloud = sr.read("boreas/scan.las", input_root="/media/sasaki/aiueo/datasets") +sr.write( + "runs/labeled.ply", + cloud, + output_root="/media/sasaki/aiueo/spatialrust-results", + manifest_path="runs/labeled.json", +) + +result = sr.run_pipeline_files( + "boreas/scan.las", + "runs/labeled.ply", + input_root="/media/sasaki/aiueo/datasets", + output_root="/media/sasaki/aiueo/spatialrust-results", + manifest_path="runs/mvp.json", +) +``` + +`open_point_cloud_stream(..., input_root=...)` applies the same input +resolution to bounded streaming. The output root only controls the destination +path supplied by the caller; no intermediate dataset copy is created. diff --git a/docs/FEATURE_MATRIX.md b/docs/FEATURE_MATRIX.md index 23732e9..1f66857 100644 --- a/docs/FEATURE_MATRIX.md +++ b/docs/FEATURE_MATRIX.md @@ -14,6 +14,7 @@ should normally depend on `spatialrust` and select only the profiles they need. | `pipeline-mvp-gpu` | `mvp` GPU stages plus `gpu-wgpu` | GPU-enabled MVP pipeline | | `gpu-aoso-staging` | AoSoA core packing plus GPU-resident frame APIs | Chained GPU execution with explicit readback | | `serde` | serde derives for core/math metadata and schemas | Configuration and metadata serialization | +| `io-manifest` | explicit storage roots plus file manifests | SHA-256/size receipts for local IO | The Python extension selects its supported meta-crate features in `crates/spatialrust-py/Cargo.toml`. It is intentionally outside the Rust @@ -25,7 +26,7 @@ workspace because its build requires a Python toolchain. | --- | --- | --- | --- | | `spatialrust-core` | schema, metadata, `PointCloud`, tensors, execution contracts | serde, AoSoA packing | none | | `spatialrust-math` | vector/matrix/pose math | serde | none | -| `spatialrust-io` | no format enabled by default | PCD, PLY, LAS/LAZ, E57, COPC, HTTP COPC | format crates are optional | +| `spatialrust-io` | no format enabled by default | PCD, PLY, LAS/LAZ, E57, COPC, HTTP COPC, explicit roots/manifests | format and checksum crates are optional | | `spatialrust-search` | KD-tree | graph, parallel queries | none | | `spatialrust-filtering` | voxel | GPU voxel, outlier, crop, FPS, MLS | wgpu/search optional | | `spatialrust-features` | normals | ISS, orientation, boundary, GPU normals | wgpu/search optional | diff --git a/docs/STREAMING_IO.md b/docs/STREAMING_IO.md index caa2091..659450a 100644 --- a/docs/STREAMING_IO.md +++ b/docs/STREAMING_IO.md @@ -53,3 +53,9 @@ cargo run -p spatialrust-io --example bounded_pcd_to_ply ` Holding more than one chunk at once is allowed, but every live lease remains charged to the shared memory budget. Drop a processed chunk before pulling the next one when the budget is sized for one chunk. + +The `spatialrust-stream` CLI additionally accepts `--input-root` and +`--output-root` for external storage, plus `--manifest PATH` for local +input/output size and SHA-256 receipts. See +`/home/sasaki/workspace/SpatialRust/docs/EXTERNAL_STORAGE.md` for the complete +path contract.