Add explicit external storage IO roots and manifests

Cargo.toml

@@ -109,6 +109,7 @@ exr = { version = "=1.72.0", default-features = false }
 tempfile = "3"
 winit = "0.30"
 serde_json = "1"
+sha2 = "0.10"
 wasm-bindgen = "0.2"
 wasm-bindgen-futures = "0.4"
 js-sys = "0.3"

README.md

@@ -641,6 +641,19 @@ let cloud = read_point_cloud_file("scan.las")?;
 write_point_cloud_file("output.ply", &cloud)?;
 ```
 
+For datasets on an external SSD, resolve logical input/output paths explicitly
+and emit a size/SHA-256 manifest:
+
+```bash
+cargo run -p spatialrust --features mvp --bin spatialrust-mvp -- \
+  --input-root /media/sasaki/aiueo/datasets \
+  --output-root /media/sasaki/aiueo/spatialrust-results \
+  --manifest runs/scan.json boreas/scan.las runs/scan.ply
+```
+
+See [`docs/EXTERNAL_STORAGE.md`](docs/EXTERNAL_STORAGE.md) for the Python and
+bounded-streaming equivalents.
+
 COPC partial read:
 
 ```rust

crates/spatialrust-io/Cargo.toml

@@ -19,6 +19,7 @@ io-copc = ["dep:copc-streaming", "dep:copc-core", "dep:copc-writer", "dep:pollst
 io-copc-http = ["io-copc", "dep:ureq"]
 streaming = ["dep:spatialrust-records"]
 serde = ["spatialrust-core/serde"]
+io-manifest = ["dep:serde", "dep:serde_json", "dep:sha2"]
 
 [dependencies]
 spatialrust-core = { workspace = true }
@@ -32,8 +33,12 @@ copc-core = { workspace = true, optional = true }
 copc-writer = { workspace = true, optional = true }
 pollster = { workspace = true, optional = true }
 ureq = { workspace = true, optional = true }
+serde = { workspace = true, optional = true }
+serde_json = { workspace = true, optional = true }
+sha2 = { workspace = true, optional = true }
 
 [dev-dependencies]
+tempfile.workspace = true
 
 [[example]]
 name = "bounded_pcd_to_ply"

crates/spatialrust-io/src/error.rs

@@ -57,6 +57,14 @@ pub enum IoError {
     #[error("streaming io error: {0}")]
     Streaming(String),
 
+    /// Input/output root or dataset path contract failure.
+    #[error("storage path error: {0}")]
+    Storage(String),
+
+    /// Dataset manifest serialization or receipt failure.
+    #[error("manifest error: {0}")]
+    Manifest(String),
+
     /// Core data model error propagated from `spatialrust-core`.
     #[error(transparent)]
     Core(#[from] SpatialError),

crates/spatialrust-io/src/lib.rs

@@ -7,9 +7,12 @@
 
 mod error;
 mod format;
+#[cfg(feature = "io-manifest")]
+mod manifest;
 mod options;
 #[cfg(feature = "streaming")]
 mod spool;
+mod storage;
 #[cfg(all(
     feature = "streaming",
     any(feature = "io-pcd", feature = "io-ply", feature = "io-las", feature = "io-copc")
@@ -42,9 +45,12 @@ pub use format::{
     detect_point_cloud_format, read_point_cloud_file, read_point_cloud_file_with_format,
     write_point_cloud_file, write_point_cloud_file_with_format, PointCloudFileFormat,
 };
+#[cfg(feature = "io-manifest")]
+pub use manifest::{DatasetManifest, FileReceipt, ReceiptRole, DATASET_MANIFEST_VERSION};
 pub use options::{ReadOptions, WriteOptions};
 #[cfg(feature = "streaming")]
 pub use spool::{BoundedSpool, SpoolOptions};
+pub use storage::StorageRoots;
 pub use traits::{PointReader, PointSink, PointStream, PointWriter};
 
 #[cfg(feature = "io-pcd")]

crates/spatialrust-io/src/manifest.rs

@@ -0,0 +1,176 @@
+//! Checksummed file receipts and dataset manifests.
+
+use std::fs::File;
+use std::io::{BufReader, Read};
+use std::path::{Path, PathBuf};
+
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+
+use crate::IoError;
+
+/// Current JSON schema version for [`DatasetManifest`].
+pub const DATASET_MANIFEST_VERSION: u32 = 1;
+
+/// Logical role of a file in a dataset operation.
+#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ReceiptRole {
+    /// File consumed by an operation.
+    Input,
+    /// File produced by an operation.
+    Output,
+    /// File associated with an operation but not consumed or produced by it.
+    Auxiliary,
+}
+
+/// Size and SHA-256 receipt for one local file or URI source.
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct FileReceipt {
+    /// Role of the source in the operation.
+    pub role: ReceiptRole,
+    /// Resolved local path or URI written as a path-like JSON string.
+    pub path: PathBuf,
+    /// Number of bytes observed while hashing a local file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub size_bytes: Option<u64>,
+    /// Lowercase hexadecimal SHA-256 digest for a local file.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub sha256: Option<String>,
+}
+
+impl FileReceipt {
+    /// Hashes a local file and returns its size/checksum receipt.
+    pub fn from_path(role: ReceiptRole, path: impl AsRef<Path>) -> Result<Self, IoError> {
+        let path = path.as_ref();
+        let file = File::open(path)?;
+        let mut reader = BufReader::new(file);
+        let mut hasher = Sha256::new();
+        let mut bytes = 0_u64;
+        let mut buffer = [0_u8; 64 * 1024];
+
+        loop {
+            let read = reader.read(&mut buffer)?;
+            if read == 0 {
+                break;
+            }
+            hasher.update(&buffer[..read]);
+            bytes = bytes.checked_add(read as u64).ok_or_else(|| {
+                IoError::Manifest(format!("file size overflow while hashing `{}`", path.display()))
+            })?;
+        }
+
+        let digest = hasher.finalize();
+        Ok(Self {
+            role,
+            path: path.to_path_buf(),
+            size_bytes: Some(bytes),
+            sha256: Some(hex_digest(&digest)),
+        })
+    }
+
+    /// Records a URI source whose bytes were not materialized locally.
+    #[must_use]
+    pub fn from_uri(role: ReceiptRole, uri: impl Into<PathBuf>) -> Self {
+        Self { role, path: uri.into(), size_bytes: None, sha256: None }
+    }
+}
+
+/// JSON manifest containing the files associated with one dataset operation.
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct DatasetManifest {
+    /// Version of the manifest JSON schema.
+    pub version: u32,
+    /// File and URI receipts in operation order.
+    pub entries: Vec<FileReceipt>,
+}
+
+impl Default for DatasetManifest {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl DatasetManifest {
+    /// Creates an empty manifest at the current schema version.
+    #[must_use]
+    pub const fn new() -> Self {
+        Self { version: DATASET_MANIFEST_VERSION, entries: Vec::new() }
+    }
+
+    /// Adds a checksummed local file receipt.
+    pub fn add_file(&mut self, role: ReceiptRole, path: impl AsRef<Path>) -> Result<(), IoError> {
+        self.entries.push(FileReceipt::from_path(role, path)?);
+        Ok(())
+    }
+
+    /// Adds a URI receipt without claiming a local byte count or checksum.
+    pub fn add_uri(&mut self, role: ReceiptRole, uri: impl Into<PathBuf>) {
+        self.entries.push(FileReceipt::from_uri(role, uri));
+    }
+
+    /// Serializes this manifest as pretty-printed JSON.
+    pub fn to_json(&self) -> Result<String, IoError> {
+        serde_json::to_string_pretty(self).map_err(|error| {
+            IoError::Manifest(format!("cannot serialize dataset manifest: {error}"))
+        })
+    }
+
+    /// Writes this manifest as JSON, creating its parent directory if needed.
+    pub fn write_json(&self, path: impl AsRef<Path>) -> Result<(), IoError> {
+        let path = path.as_ref();
+        if let Some(parent) = path.parent() {
+            if !parent.as_os_str().is_empty() {
+                std::fs::create_dir_all(parent)?;
+            }
+        }
+        std::fs::write(path, format!("{}\n", self.to_json()?))?;
+        Ok(())
+    }
+}
+
+fn hex_digest(bytes: &[u8]) -> String {
+    let mut output = String::with_capacity(bytes.len() * 2);
+    for byte in bytes {
+        output.push_str(&format!("{byte:02x}"));
+    }
+    output
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{DatasetManifest, FileReceipt, ReceiptRole, DATASET_MANIFEST_VERSION};
+
+    #[test]
+    fn hashes_file_and_records_size() {
+        let directory = tempfile::tempdir().unwrap();
+        let path = directory.path().join("scan.bin");
+        std::fs::write(&path, b"hello").unwrap();
+
+        let receipt = FileReceipt::from_path(ReceiptRole::Input, &path).unwrap();
+        assert_eq!(receipt.size_bytes, Some(5));
+        assert_eq!(
+            receipt.sha256.as_deref(),
+            Some("2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824")
+        );
+    }
+
+    #[test]
+    fn serializes_local_and_uri_entries() {
+        let directory = tempfile::tempdir().unwrap();
+        let path = directory.path().join("scan.bin");
+        std::fs::write(&path, b"data").unwrap();
+
+        let mut manifest = DatasetManifest::new();
+        manifest.add_file(ReceiptRole::Input, &path).unwrap();
+        manifest.add_uri(ReceiptRole::Auxiliary, "https://example.test/scan.copc.laz");
+        let json = manifest.to_json().unwrap();
+        let decoded: DatasetManifest = serde_json::from_str(&json).unwrap();
+
+        assert_eq!(decoded.version, DATASET_MANIFEST_VERSION);
+        assert_eq!(decoded.entries.len(), 2);
+        assert_eq!(decoded.entries[0].role, ReceiptRole::Input);
+        assert_eq!(decoded.entries[0].size_bytes, Some(4));
+        assert_eq!(decoded.entries[1].sha256, None);
+    }
+}

crates/spatialrust-io/src/storage.rs

@@ -0,0 +1,111 @@
+//! Explicit input/output roots for local spatial data.
+
+use std::path::{Component, Path, PathBuf};
+
+use crate::IoError;
+
+/// Optional roots used to resolve logical input and output paths.
+///
+/// Relative paths are joined to their corresponding root. Absolute paths are
+/// always treated as explicit locations and bypass the roots. Relative paths
+/// containing `..` are rejected so a logical dataset path cannot escape its
+/// configured root.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct StorageRoots {
+    input_root: Option<PathBuf>,
+    output_root: Option<PathBuf>,
+}
+
+impl StorageRoots {
+    /// Creates a root mapping. Either root may be omitted.
+    #[must_use]
+    pub const fn new(input_root: Option<PathBuf>, output_root: Option<PathBuf>) -> Self {
+        Self { input_root, output_root }
+    }
+
+    /// Returns the configured input root, if any.
+    #[must_use]
+    pub fn input_root(&self) -> Option<&Path> {
+        self.input_root.as_deref()
+    }
+
+    /// Returns the configured output root, if any.
+    #[must_use]
+    pub fn output_root(&self) -> Option<&Path> {
+        self.output_root.as_deref()
+    }
+
+    /// Resolves a logical input path against the configured input root.
+    pub fn resolve_input(&self, path: impl AsRef<Path>) -> Result<PathBuf, IoError> {
+        resolve_path(self.input_root.as_deref(), path.as_ref(), "input")
+    }
+
+    /// Resolves a logical output path against the configured output root.
+    pub fn resolve_output(&self, path: impl AsRef<Path>) -> Result<PathBuf, IoError> {
+        resolve_path(self.output_root.as_deref(), path.as_ref(), "output")
+    }
+
+    /// Creates the parent directory for an explicit output path when needed.
+    pub fn ensure_output_parent(&self, path: impl AsRef<Path>) -> Result<(), IoError> {
+        if let Some(parent) = path.as_ref().parent() {
+            if !parent.as_os_str().is_empty() {
+                std::fs::create_dir_all(parent)?;
+            }
+        }
+        Ok(())
+    }
+}
+
+fn resolve_path(root: Option<&Path>, path: &Path, kind: &str) -> Result<PathBuf, IoError> {
+    if path.is_absolute() || root.is_none() {
+        return Ok(path.to_path_buf());
+    }
+
+    if path.components().any(|component| component == Component::ParentDir) {
+        return Err(IoError::Storage(format!(
+            "relative {kind} path `{}` must not contain `..`",
+            path.display()
+        )));
+    }
+
+    Ok(root.expect("root checked above").join(path))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::StorageRoots;
+    use std::path::PathBuf;
+
+    #[test]
+    fn resolves_relative_paths_per_direction() {
+        let roots = StorageRoots::new(
+            Some(PathBuf::from("/mnt/input")),
+            Some(PathBuf::from("/mnt/output")),
+        );
+        assert_eq!(roots.resolve_input("scan.las").unwrap(), PathBuf::from("/mnt/input/scan.las"));
+        assert_eq!(
+            roots.resolve_output("runs/result.las").unwrap(),
+            PathBuf::from("/mnt/output/runs/result.las")
+        );
+    }
+
+    #[test]
+    fn absolute_paths_bypass_roots() {
+        let roots = StorageRoots::new(
+            Some(PathBuf::from("/mnt/input")),
+            Some(PathBuf::from("/mnt/output")),
+        );
+        assert_eq!(roots.resolve_input("/tmp/scan.las").unwrap(), PathBuf::from("/tmp/scan.las"));
+        assert_eq!(
+            roots.resolve_output("/tmp/result.las").unwrap(),
+            PathBuf::from("/tmp/result.las")
+        );
+    }
+
+    #[test]
+    fn rejects_relative_root_escape() {
+        let roots = StorageRoots::new(Some(PathBuf::from("/mnt/input")), None);
+        let error = roots.resolve_input("../private/scan.las").unwrap_err();
+        assert!(error.to_string().contains("must not contain `..`"));
+    }
+}