diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39314fd..e7f3fc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -256,6 +256,7 @@ jobs: . .venv/bin/activate pip install maturin pytest numpy mypy maturin develop --release --manifest-path crates/spatialrust-py/Cargo.toml + python crates/spatialrust-py/examples/video_tracking_e2e.py --no-gif pytest crates/spatialrust-py/tests # The comparison report contract stays stdlib-only. OpenCV itself is # intentionally not a production or default-CI dependency. @@ -285,6 +286,7 @@ jobs: cargo run -p spatialrust --no-default-features --features vision-full --example vision_1_cpu cargo run -p spatialrust --no-default-features --features platform --example vision_1_release_gate cargo run -p spatialrust --no-default-features --features platform --example vision_2_release_gate + cargo run -p spatialrust --no-default-features --features image-io-standard,vision-video --example video_tracking_e2e bench: name: Benchmark compile diff --git a/README.md b/README.md index 71888da..9ddec98 100644 --- a/README.md +++ b/README.md @@ -479,6 +479,23 @@ The reproducible algorithm comparison is in `bench/opencv_vision_comparison/`; the complete synthetic demo is `crates/spatialrust-py/examples/vision_ai_pipeline.py`. +The video E2E demo generates and reloads the same deterministic 12-frame PGM +sequence in Rust and Python, estimates dense optical flow, detects the two +moving objects, and preserves track IDs through the native IoU tracker: + +

+ Two textured objects moving in opposite directions with SpatialRust dense optical-flow vectors and stable track IDs 1 and 2 +

+ +```powershell +cargo run -p spatialrust --no-default-features --features image-io-standard,vision-video --example video_tracking_e2e +maturin develop --release --manifest-path crates/spatialrust-py/Cargo.toml +.venv/Scripts/python.exe crates/spatialrust-py/examples/video_tracking_e2e.py +``` + +Both paths assert object-center flow `(+2,+1)` / `(-2,-1)` for all 11 frame +pairs and stable track IDs `1,2`. The Python run regenerates the GIF above. + The same feature includes Harris, Shi–Tomasi, exact FAST-9/16, multi-scale ORB, and checked Hamming/L2 descriptor matching. Python exposes `orb_features` and NumPy matcher functions; OpenCV is used only by the numerical comparison suite. diff --git a/crates/spatialrust-py/examples/video_tracking_e2e.py b/crates/spatialrust-py/examples/video_tracking_e2e.py new file mode 100644 index 0000000..6e513a2 --- /dev/null +++ b/crates/spatialrust-py/examples/video_tracking_e2e.py @@ -0,0 +1,179 @@ +"""Frame sequence -> dense optical flow -> deterministic object tracking. + +The input is generated deterministically as portable PGM files, then loaded +through SpatialRust's bounded image IO. The same sequence is consumed by the +Rust `video_tracking_e2e` example. +""" + +from __future__ import annotations + +import argparse +from collections import deque +from pathlib import Path + +import numpy as np +import spatialrust as sr + + +WIDTH = 96 +HEIGHT = 72 +FRAME_COUNT = 12 + + +def paint_object(image, x0, y0, width, height, base): + for local_y in range(height): + for local_x in range(width): + image[y0 + local_y, x0 + local_x] = ( + base + (local_x * 7 + local_y * 11 + local_x * local_y * 3) % 25 + ) + + +def generate_frames(directory: Path) -> None: + directory.mkdir(parents=True, exist_ok=True) + yy, xx = np.indices((HEIGHT, WIDTH), dtype=np.int32) + background = (20 + (xx * 3 + yy * 5) % 20).astype(np.uint8) + for index in range(FRAME_COUNT): + image = background.copy() + paint_object(image, 8 + index * 2, 9 + index, 18, 14, 150) + paint_object(image, 70 - index * 2, 46 - index, 16, 12, 220) + path = directory / f"frame_{index:02}.pgm" + path.write_bytes(f"P5\n{WIDTH} {HEIGHT}\n255\n".encode() + image.tobytes()) + + +def load_frames(directory: Path) -> list[np.ndarray]: + frames = [] + for path in sorted(directory.glob("frame_*.pgm")): + image, _metadata = sr.read_image(str(path)) + if image.ndim != 2 or image.dtype != np.uint8: + raise RuntimeError(f"{path} did not decode as Gray8") + frames.append(image) + if len(frames) < 2: + raise RuntimeError("need at least two generated frames") + return frames + + +def detect_objects(image: np.ndarray): + foreground = image >= 100 + visited = np.zeros_like(foreground, dtype=bool) + boxes = [] + classes = [] + for y0, x0 in np.argwhere(foreground): + if visited[y0, x0]: + continue + visited[y0, x0] = True + queue = deque([(int(x0), int(y0))]) + xs = [] + ys = [] + maximum = 0 + while queue: + x, y = queue.popleft() + xs.append(x) + ys.append(y) + maximum = max(maximum, int(image[y, x])) + for nx, ny in ((x - 1, y), (x + 1, y), (x, y - 1), (x, y + 1)): + if ( + 0 <= nx < image.shape[1] + and 0 <= ny < image.shape[0] + and foreground[ny, nx] + and not visited[ny, nx] + ): + visited[ny, nx] = True + queue.append((nx, ny)) + if len(xs) >= 20: + boxes.append([min(xs), min(ys), max(xs) + 1, max(ys) + 1]) + classes.append(2 if maximum >= 210 else 1) + order = np.argsort(classes) + return ( + np.asarray(boxes, dtype=np.float32)[order], + np.ones(len(boxes), dtype=np.float32)[order], + np.asarray(classes, dtype=np.int64)[order], + ) + + +def render_gif(frames, tracks_per_frame, flows, output: Path) -> None: + from PIL import Image, ImageDraw + + colors = {1: (255, 91, 91), 2: (87, 211, 176)} + rendered = [] + for index, (frame, tracks) in enumerate(zip(frames, tracks_per_frame)): + canvas = Image.fromarray(frame, mode="L").convert("RGB").resize((WIDTH * 4, HEIGHT * 4)) + draw = ImageDraw.Draw(canvas) + for row in tracks: + track_id = int(row[0]) + box = [int(round(value * 4)) for value in row[1:5]] + class_id = int(row[5]) + color = colors[class_id] + draw.rectangle(box, outline=color, width=3) + draw.text((box[0] + 3, box[1] + 3), f"id {track_id}", fill=color) + if index > 0: + flow = flows[index - 1] + for y in range(8, HEIGHT - 8, 12): + for x in range(8, WIDTH - 8, 12): + dx, dy = flow[y, x] + if np.isfinite(dx) and np.isfinite(dy) and abs(dx) + abs(dy) >= 1: + draw.line( + (x * 4, y * 4, (x + float(dx) * 2) * 4, (y + float(dy) * 2) * 4), + fill=(255, 207, 112), + width=2, + ) + draw.text((8, 8), f"frame {index:02}", fill=(240, 245, 255)) + rendered.append(canvas) + output.parent.mkdir(parents=True, exist_ok=True) + rendered[0].save( + output, + save_all=True, + append_images=rendered[1:], + duration=140, + loop=0, + optimize=False, + ) + + +def run(frames_dir: Path, gif_path: Path, render: bool = True) -> None: + generate_frames(frames_dir) + frames = load_frames(frames_dir) + tracker = sr.MultiObjectTracker(iou_threshold=0.2, max_missed=1, min_confirmed_hits=2) + tracks_per_frame = [] + flows = [] + boxes, scores, classes = detect_objects(frames[0]) + tracks_per_frame.append(tracker.update(boxes, scores, classes)) + for previous, current in zip(frames, frames[1:]): + flow = sr.dense_flow_image(previous, current, block_radius=1, search_radius=3) + previous_boxes, _previous_scores, previous_classes = detect_objects(previous) + observed = [] + for box, class_id in zip(previous_boxes, previous_classes): + x = int((box[0] + box[2]) * 0.5) + y = int((box[1] + box[3]) * 0.5) + observed.append((int(class_id), *flow[y, x].tolist())) + if observed != [(1, 2.0, 1.0), (2, -2.0, -1.0)]: + raise RuntimeError(f"unexpected object flow: {observed}") + flows.append(flow) + boxes, scores, classes = detect_objects(current) + tracks = tracker.update(boxes, scores, classes) + if len(tracks) != 2 or [track[0] for track in tracks] != [1, 2]: + raise RuntimeError(f"unstable tracks: {tracks}") + tracks_per_frame.append(tracks) + if render: + render_gif(frames, tracks_per_frame, flows, gif_path) + print( + f"video_tracking_e2e=ok frames={len(frames)} " + f"flow_pairs={len(flows)} stable_track_ids=1,2 " + f"gif={gif_path if render else 'skipped'}" + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--frames-dir", type=Path, default=Path("target/video-tracking-demo/frames") + ) + parser.add_argument( + "--gif", type=Path, default=Path("docs/assets/video_tracking_e2e.gif") + ) + parser.add_argument("--no-gif", action="store_true") + args = parser.parse_args() + run(args.frames_dir, args.gif, render=not args.no_gif) + + +if __name__ == "__main__": + main() diff --git a/crates/spatialrust-py/spatialrust.pyi b/crates/spatialrust-py/spatialrust.pyi index e3e047e..5a4fc5c 100644 --- a/crates/spatialrust-py/spatialrust.pyi +++ b/crates/spatialrust-py/spatialrust.pyi @@ -15,7 +15,7 @@ __all__: list[str] = [ "__version__", "ImageMetadata", "Tensor", "Keypoint2", "OnnxRuntimeSession", "DLPackTensorView", "PointCloud", "PipelineResult", "RegionResult", "DbscanResult", "GroundResult", "MultiPlaneResult", "SphereResult", - "CylinderResult", "RegistrationResult", "read_image", + "CylinderResult", "RegistrationResult", "MultiObjectTracker", "read_image", "tensor_copy_from_numpy", "tensor_view_from_dlpack", "harris_keypoints", "shi_tomasi_keypoints", "fast_keypoints", "orb_features", "estimate_homography_ransac", "solve_pnp", "estimate_rgbd_odometry", "stereo_block_match", @@ -140,6 +140,27 @@ def dense_flow_image( search_radius: int = ..., ) -> _F32Array: ... +@final +class MultiObjectTracker: + """Deterministic same-class IoU tracker with persistent track IDs.""" + + def __new__( + cls, + iou_threshold: float = ..., + max_missed: int = ..., + min_confirmed_hits: int = ..., + ) -> MultiObjectTracker: ... + def update( + self, + boxes: _F32Array, + scores: _F32Array, + class_ids: NDArray[np.int64], + ) -> list[ + tuple[int, float, float, float, float, int, float, int, int, int, bool] + ]: + """Return ``id, box, class, score, age, hits, missed, confirmed`` tuples.""" + ... + def rgbd_to_point_cloud( depth: _F32Array, color: _U8Array, diff --git a/crates/spatialrust-py/src/lib.rs b/crates/spatialrust-py/src/lib.rs index a55eaff..f8f6227 100644 --- a/crates/spatialrust-py/src/lib.rs +++ b/crates/spatialrust-py/src/lib.rs @@ -112,9 +112,12 @@ use spatialrust::vision::{ ObjectImageCorrespondence, OrbOptions, OrbScoreType, PanoramaOptions, PerspectiveTransform, PointCorrespondence2, PointMap, RectMorphologyWorkspace, RgbdOdometryOptions, RleOrder, RobustEstimationOptions, ShiTomasiOptions, SoftNmsMethod, StereoBmOptions, StructuringElement, - ThresholdType, + ThresholdType, TrackState, +}; +use spatialrust::vision::{ + dense_flow_block_match as dense_flow_native, DenseFlowOptions, + MultiObjectTracker as NativeMultiObjectTracker, MultiObjectTrackerOptions, }; -use spatialrust::vision::{dense_flow_block_match as dense_flow_native, DenseFlowOptions}; use spatialrust::voxelize::{ range_image as range_image_proj, voxelize as voxelize_grid, RangeImageConfig, VoxelFill, VoxelGridConfig, @@ -134,6 +137,7 @@ use spatialrust::{ type Vec3Tuple = (f32, f32, f32); type OrientedBoundingBoxTuple = (Vec3Tuple, Vec3Tuple, Vec); +type ObjectTrackTuple = (u64, f32, f32, f32, f32, i64, f32, u32, u32, u32, bool); type ComponentStats = Vec<(u32, usize, (f32, f32, f32, f32))>; fn to_py_err(err: E) -> PyErr { @@ -824,6 +828,81 @@ fn dense_flow_image<'py>( Ok(array.into_pyarray_bound(py)) } +/// Stateful deterministic same-class IoU tracker. +#[pyclass(name = "MultiObjectTracker")] +struct PyMultiObjectTracker { + inner: NativeMultiObjectTracker, +} + +#[pymethods] +impl PyMultiObjectTracker { + /// Creates an empty tracker. + #[new] + #[pyo3(signature = (iou_threshold=0.3, max_missed=3, min_confirmed_hits=2))] + fn new(iou_threshold: f32, max_missed: u32, min_confirmed_hits: u32) -> PyResult { + let inner = NativeMultiObjectTracker::try_new(MultiObjectTrackerOptions { + iou_threshold, + max_missed, + min_confirmed_hits, + }) + .map_err(to_py_err)?; + Ok(Self { inner }) + } + + /// Associates detections and returns rows ending in a confirmed 0/1 field. + fn update( + &mut self, + boxes: PyReadonlyArray2<'_, f32>, + scores: PyReadonlyArray1<'_, f32>, + class_ids: PyReadonlyArray1<'_, i64>, + ) -> PyResult> { + let boxes = boxes.as_array(); + let scores = scores.as_array(); + let class_ids = class_ids.as_array(); + if boxes.ndim() != 2 + || boxes.shape()[1] != 4 + || scores.len() != boxes.shape()[0] + || class_ids.len() != boxes.shape()[0] + { + return Err(PyValueError::new_err( + "boxes must be Nx4 with matching scores and class_ids", + )); + } + let detections = boxes + .outer_iter() + .zip(scores.iter()) + .zip(class_ids.iter()) + .map(|((row, &score), &class_id)| { + Ok(Detection { + bbox: BoundingBox2::try_new(row[0], row[1], row[2], row[3]) + .map_err(to_py_err)?, + score, + class_id, + }) + }) + .collect::>>()?; + let tracks = self.inner.update(&detections).map_err(to_py_err)?; + Ok(tracks + .iter() + .map(|track| { + ( + track.id, + track.bbox.x_min, + track.bbox.y_min, + track.bbox.x_max, + track.bbox.y_max, + track.class_id, + track.score, + track.age, + track.hits, + track.missed, + track.state == TrackState::Confirmed, + ) + }) + .collect()) + } +} + /// Applies gray-world white balance to an RGB image. #[pyfunction] fn gray_world_white_balance_image<'py>( @@ -4162,6 +4241,7 @@ fn spatialrust_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/crates/spatialrust-py/tests/test_bindings.py b/crates/spatialrust-py/tests/test_bindings.py index 8a10a68..0ec1af9 100644 --- a/crates/spatialrust-py/tests/test_bindings.py +++ b/crates/spatialrust-py/tests/test_bindings.py @@ -1036,3 +1036,19 @@ def test_dense_flow_binding_recovers_translation_and_marks_border_invalid(): assert flow.shape == (height, width, 2) np.testing.assert_array_equal(flow[16, 20], [2.0, 1.0]) assert np.isnan(flow[0, 0]).all() + + +def test_multi_object_tracker_preserves_ids_and_confirms(): + tracker = sr.MultiObjectTracker(iou_threshold=0.2, max_missed=1, min_confirmed_hits=2) + scores = np.array([0.9], dtype=np.float32) + classes = np.array([3], dtype=np.int64) + first = tracker.update( + np.array([[10.0, 10.0, 24.0, 22.0]], dtype=np.float32), scores, classes + ) + second = tracker.update( + np.array([[12.0, 11.0, 26.0, 23.0]], dtype=np.float32), scores, classes + ) + assert len(first) == len(second) == 1 + assert first[0][0] == second[0][0] == 1 + assert first[0][10] is False + assert second[0][10] is True diff --git a/crates/spatialrust/Cargo.toml b/crates/spatialrust/Cargo.toml index 7878748..57beabf 100644 --- a/crates/spatialrust/Cargo.toml +++ b/crates/spatialrust/Cargo.toml @@ -276,6 +276,11 @@ name = "vision_2_release_gate" path = "examples/vision_2_release_gate.rs" required-features = ["platform"] +[[example]] +name = "video_tracking_e2e" +path = "examples/video_tracking_e2e.rs" +required-features = ["image-io-standard", "vision-video"] + [[bin]] name = "spatialrust-mvp" path = "src/bin/spatialrust_mvp.rs" diff --git a/crates/spatialrust/examples/video_tracking_e2e.rs b/crates/spatialrust/examples/video_tracking_e2e.rs new file mode 100644 index 0000000..344bc4e --- /dev/null +++ b/crates/spatialrust/examples/video_tracking_e2e.rs @@ -0,0 +1,211 @@ +//! Loads a deterministic frame sequence, estimates optical flow, and tracks objects. + +use std::collections::VecDeque; +use std::path::{Path, PathBuf}; + +use spatialrust::image_io::{decode_path, DecodeOptions, DecodedPixels}; +use spatialrust::vision::{ + dense_flow_block_match, BoundingBox2, DenseFlowOptions, Detection, MultiObjectTracker, + MultiObjectTrackerOptions, ObjectTrack, +}; +use spatialrust::Image; + +fn main() { + let frame_dir = std::env::args() + .nth(1) + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("target/video-tracking-demo/frames")); + generate_frames(&frame_dir); + let frames = load_frames(&frame_dir); + assert!(frames.len() >= 2, "need at least two generated PGM frames"); + + let mut tracker = MultiObjectTracker::try_new(MultiObjectTrackerOptions { + iou_threshold: 0.2, + max_missed: 1, + min_confirmed_hits: 2, + }) + .expect("tracker options"); + let first_detections = detect_objects(&frames[0]); + tracker.update(&first_detections).expect("first detections"); + + let mut flow_pairs = 0usize; + for (index, pair) in frames.windows(2).enumerate() { + let flow = dense_flow_block_match( + pair[0].view(), + pair[1].view(), + DenseFlowOptions { block_radius: 1, search_radius: 3, minimum_improvement: 1 }, + ) + .expect("dense optical flow"); + let previous_detections = detect_objects(&pair[0]); + let detections = detect_objects(&pair[1]); + let tracks = tracker.update(&detections).expect("tracking update"); + let object_flows = detection_center_flows(&flow, &previous_detections); + assert_eq!(object_flows, vec![(1, 2.0, 1.0), (2, -2.0, -1.0)]); + println!( + "frame={:02} detections={} tracks={} flow_class1=(2,1) flow_class2=(-2,-1)", + index + 1, + detections.len(), + tracks.len() + ); + assert_stable_tracks(tracks); + flow_pairs += 1; + } + assert_eq!(flow_pairs, frames.len() - 1); + assert_eq!(tracker.tracks().len(), 2); + println!( + "video_tracking_e2e=ok frames={} flow_pairs={} stable_track_ids=1,2", + frames.len(), + flow_pairs + ); +} + +fn generate_frames(directory: &Path) { + std::fs::create_dir_all(directory) + .unwrap_or_else(|error| panic!("create {}: {error}", directory.display())); + for frame_index in 0..12usize { + let width = 96usize; + let height = 72usize; + let mut pixels = vec![0u8; width * height]; + for y in 0..height { + for x in 0..width { + pixels[y * width + x] = 20 + ((x * 3 + y * 5) % 20) as u8; + } + } + paint_object(&mut pixels, width, 8 + frame_index * 2, 9 + frame_index, 18, 14, 150); + paint_object(&mut pixels, width, 70 - frame_index * 2, 46 - frame_index, 16, 12, 220); + let path = directory.join(format!("frame_{frame_index:02}.pgm")); + let mut encoded = format!("P5\n{width} {height}\n255\n").into_bytes(); + encoded.extend_from_slice(&pixels); + std::fs::write(&path, encoded) + .unwrap_or_else(|error| panic!("write {}: {error}", path.display())); + } +} + +fn paint_object( + pixels: &mut [u8], + width: usize, + x0: usize, + y0: usize, + object_width: usize, + object_height: usize, + base: u8, +) { + for local_y in 0..object_height { + for local_x in 0..object_width { + let x = x0 + local_x; + let y = y0 + local_y; + pixels[y * width + x] = + base + ((local_x * 7 + local_y * 11 + local_x * local_y * 3) % 25) as u8; + } + } +} + +fn load_frames(directory: &Path) -> Vec> { + let mut paths = std::fs::read_dir(directory) + .unwrap_or_else(|error| panic!("read {}: {error}", directory.display())) + .map(|entry| entry.expect("frame directory entry").path()) + .filter(|path| path.extension().is_some_and(|extension| extension == "pgm")) + .collect::>(); + paths.sort(); + paths + .into_iter() + .map(|path| { + let decoded = decode_path(&path, DecodeOptions::default()) + .unwrap_or_else(|error| panic!("decode {}: {error}", path.display())); + match decoded.into_pixels() { + DecodedPixels::Gray8(image) => image, + _ => panic!("{} is not Gray8", path.display()), + } + }) + .collect() +} + +fn detect_objects(image: &Image) -> Vec { + let width = image.width(); + let height = image.height(); + let pixels = image.as_slice(); + let mut visited = vec![false; pixels.len()]; + let mut detections = Vec::new(); + for start in 0..pixels.len() { + if visited[start] || pixels[start] < 100 { + continue; + } + visited[start] = true; + let mut queue = VecDeque::from([start]); + let mut min_x = width; + let mut min_y = height; + let mut max_x = 0usize; + let mut max_y = 0usize; + let mut maximum = 0u8; + let mut area = 0usize; + while let Some(index) = queue.pop_front() { + let x = index % width; + let y = index / width; + min_x = min_x.min(x); + min_y = min_y.min(y); + max_x = max_x.max(x); + max_y = max_y.max(y); + maximum = maximum.max(pixels[index]); + area += 1; + for neighbor in neighbors(x, y, width, height) { + if !visited[neighbor] && pixels[neighbor] >= 100 { + visited[neighbor] = true; + queue.push_back(neighbor); + } + } + } + if area >= 20 { + detections.push(Detection { + bbox: BoundingBox2::try_new( + min_x as f32, + min_y as f32, + (max_x + 1) as f32, + (max_y + 1) as f32, + ) + .expect("component box"), + score: 1.0, + class_id: if maximum >= 210 { 2 } else { 1 }, + }); + } + } + detections.sort_by_key(|detection| detection.class_id); + detections +} + +fn neighbors(x: usize, y: usize, width: usize, height: usize) -> Vec { + let mut result = Vec::with_capacity(4); + if x > 0 { + result.push(y * width + x - 1); + } + if x + 1 < width { + result.push(y * width + x + 1); + } + if y > 0 { + result.push((y - 1) * width + x); + } + if y + 1 < height { + result.push((y + 1) * width + x); + } + result +} + +fn detection_center_flows( + flow: &spatialrust::vision::FlowField, + detections: &[Detection], +) -> Vec<(i64, f32, f32)> { + detections + .iter() + .map(|detection| { + let x = ((detection.bbox.x_min + detection.bbox.x_max) * 0.5) as usize; + let y = ((detection.bbox.y_min + detection.bbox.y_max) * 0.5) as usize; + let vector = flow.image().get(x, y).expect("box center inside flow"); + (detection.class_id, vector[0], vector[1]) + }) + .collect() +} + +fn assert_stable_tracks(tracks: &[ObjectTrack]) { + assert_eq!(tracks.len(), 2); + assert_eq!(tracks[0].id, 1); + assert_eq!(tracks[1].id, 2); +} diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 70738fd..36acdc3 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -682,6 +682,12 @@ to one implicitly, and GPU receipts must retain named upload/readback stages. | 120D | Complete | Generated algorithm/performance documentation and migration guidance | Pages-generated receipt plus README/migration links | | 120E | Complete | Vision 2 release gate and runnable receipt example | aggregate missing/skip/duplicate/budget denial tests | +### Vision 2 video pipeline E2E demo + +| Slice | Status | Scope | Evidence | +| --- | --- | --- | --- | +| Video E2E | Complete | Deterministic frame generation/loading → dense optical flow → object detection → native multi-object tracking in Rust and Python | 12 byte-identical PGM frames, 11 exact bidirectional flow checks, stable IDs 1/2, committed GIF | + The improvement thresholds above compare against the checked Epic 112 SpatialRust baseline on the same host; they are not claims against every OpenCV build. Accuracy gates remain workload-specific: resize/gray/Gaussian retain diff --git a/docs/assets/video_tracking_e2e.gif b/docs/assets/video_tracking_e2e.gif new file mode 100644 index 0000000..8aa4ed1 Binary files /dev/null and b/docs/assets/video_tracking_e2e.gif differ diff --git a/notes/2026-07-16_video_tracking_e2e.md b/notes/2026-07-16_video_tracking_e2e.md new file mode 100644 index 0000000..37eb8aa --- /dev/null +++ b/notes/2026-07-16_video_tracking_e2e.md @@ -0,0 +1,65 @@ +# Rust/Python video tracking E2E demo — 2026-07-16 + +## Scope + +This final Vision 2 demo exercises one reproducible pipeline in both language +surfaces: + +`generate PGM sequence → load frames → dense optical flow → threshold components → native IoU tracking → GIF` + +The Rust example is +`C:\Users\rsasa\Workspace\SpatialRust\crates\spatialrust\examples\video_tracking_e2e.rs`. +The Python example and GIF renderer is +`C:\Users\rsasa\Workspace\SpatialRust\crates\spatialrust-py\examples\video_tracking_e2e.py`. +The committed output is +`C:\Users\rsasa\Workspace\SpatialRust\docs\assets\video_tracking_e2e.gif`. + +## Reproducible input + +Both examples independently generate 12 Gray8 PGM frames at 96×72. The +background and both object textures are deterministic integer formulas. Object +class 1 translates by `(+2,+1)` pixels per frame; class 2 translates by +`(-2,-1)`. + +The Rust output under `target/video-tracking-demo/frames` and Python output +under `target/video-tracking-demo/python-frames` contained 12 files each. The +ordered SHA-256 sequences were identical for all 12 files. + +No external dataset, network download, codec runtime, or random seed is needed. + +## Pipeline acceptance + +- SpatialRust bounded image IO reloads every generated PGM as Gray8. +- `dense_flow_block_match` uses block radius 1 and search radius 3. +- Every one of the 11 frame pairs reports exact object-center vectors + `(class 1, +2, +1)` and `(class 2, -2, -1)`. +- Thresholded connected components produce exactly two detections per frame. +- `MultiObjectTracker` preserves IDs 1 and 2 through the complete sequence. +- The Python binding exposes the same stateful native tracker and preserves + integer IDs/classes plus float boxes/scores in typed track tuples. +- The committed GIF is 384×288, 12 frames, and 140 ms per frame. + +## Verification + +```powershell +$env:PATH = "C:\Users\rsasa\.cargo\bin;$env:PATH" +cargo run -p spatialrust --no-default-features --features image-io-standard,vision-video --example video_tracking_e2e +cargo check --manifest-path crates/spatialrust-py/Cargo.toml +maturin develop --release --manifest-path crates/spatialrust-py/Cargo.toml +.venv/Scripts/python.exe -m pytest crates/spatialrust-py/tests/test_bindings.py -k "dense_flow or multi_object_tracker" -q +.venv/Scripts/python.exe -m mypy.stubtest spatialrust --ignore-missing-stub +.venv/Scripts/python.exe crates/spatialrust-py/examples/video_tracking_e2e.py --frames-dir target/video-tracking-demo/python-frames --gif docs/assets/video_tracking_e2e.gif +``` + +Observed results: the Rust and Python E2E receipts both reported 12 frames, 11 +flow pairs, and stable track IDs 1/2; the two focused Python tests passed; and +stubtest reported no issues. + +CI runs the Python example with `--no-gif` on Python 3.8 and 3.12, and runs the +Rust example in the Linux/Windows/macOS Vision conformance matrix. GIF rendering +remains an explicit documentation step requiring Pillow. + +The standalone `cargo clippy --manifest-path crates/spatialrust-py/Cargo.toml +-- -D warnings` command still encounters the repository's pre-existing +thread-local and morphology argument-count warnings. This slice adds no new +clippy warning.