Add Rust and Python video tracking E2E demo

.github/workflows/ci.yml

@@ -256,6 +256,7 @@ jobs:
           . .venv/bin/activate
           pip install maturin pytest numpy mypy
           maturin develop --release --manifest-path crates/spatialrust-py/Cargo.toml
+          python crates/spatialrust-py/examples/video_tracking_e2e.py --no-gif
           pytest crates/spatialrust-py/tests
           # The comparison report contract stays stdlib-only. OpenCV itself is
           # intentionally not a production or default-CI dependency.
@@ -285,6 +286,7 @@ jobs:
           cargo run -p spatialrust --no-default-features --features vision-full --example vision_1_cpu
           cargo run -p spatialrust --no-default-features --features platform --example vision_1_release_gate
           cargo run -p spatialrust --no-default-features --features platform --example vision_2_release_gate
+          cargo run -p spatialrust --no-default-features --features image-io-standard,vision-video --example video_tracking_e2e
 
   bench:
     name: Benchmark compile

README.md

@@ -479,6 +479,23 @@ The reproducible algorithm comparison is in
 `bench/opencv_vision_comparison/`; the complete synthetic demo is
 `crates/spatialrust-py/examples/vision_ai_pipeline.py`.
 
+The video E2E demo generates and reloads the same deterministic 12-frame PGM
+sequence in Rust and Python, estimates dense optical flow, detects the two
+moving objects, and preserves track IDs through the native IoU tracker:
+
+<p align="center">
+  <img src="docs/assets/video_tracking_e2e.gif" alt="Two textured objects moving in opposite directions with SpatialRust dense optical-flow vectors and stable track IDs 1 and 2" width="576">
+</p>
+
+```powershell
+cargo run -p spatialrust --no-default-features --features image-io-standard,vision-video --example video_tracking_e2e
+maturin develop --release --manifest-path crates/spatialrust-py/Cargo.toml
+.venv/Scripts/python.exe crates/spatialrust-py/examples/video_tracking_e2e.py
+```
+
+Both paths assert object-center flow `(+2,+1)` / `(-2,-1)` for all 11 frame
+pairs and stable track IDs `1,2`. The Python run regenerates the GIF above.
+
 The same feature includes Harris, Shi–Tomasi, exact FAST-9/16, multi-scale ORB,
 and checked Hamming/L2 descriptor matching. Python exposes `orb_features` and
 NumPy matcher functions; OpenCV is used only by the numerical comparison suite.

crates/spatialrust-py/examples/video_tracking_e2e.py

@@ -0,0 +1,179 @@
+"""Frame sequence -> dense optical flow -> deterministic object tracking.
+
+The input is generated deterministically as portable PGM files, then loaded
+through SpatialRust's bounded image IO. The same sequence is consumed by the
+Rust `video_tracking_e2e` example.
+"""
+
+from __future__ import annotations
+
+import argparse
+from collections import deque
+from pathlib import Path
+
+import numpy as np
+import spatialrust as sr
+
+
+WIDTH = 96
+HEIGHT = 72
+FRAME_COUNT = 12
+
+
+def paint_object(image, x0, y0, width, height, base):
+    for local_y in range(height):
+        for local_x in range(width):
+            image[y0 + local_y, x0 + local_x] = (
+                base + (local_x * 7 + local_y * 11 + local_x * local_y * 3) % 25
+            )
+
+
+def generate_frames(directory: Path) -> None:
+    directory.mkdir(parents=True, exist_ok=True)
+    yy, xx = np.indices((HEIGHT, WIDTH), dtype=np.int32)
+    background = (20 + (xx * 3 + yy * 5) % 20).astype(np.uint8)
+    for index in range(FRAME_COUNT):
+        image = background.copy()
+        paint_object(image, 8 + index * 2, 9 + index, 18, 14, 150)
+        paint_object(image, 70 - index * 2, 46 - index, 16, 12, 220)
+        path = directory / f"frame_{index:02}.pgm"
+        path.write_bytes(f"P5\n{WIDTH} {HEIGHT}\n255\n".encode() + image.tobytes())
+
+
+def load_frames(directory: Path) -> list[np.ndarray]:
+    frames = []
+    for path in sorted(directory.glob("frame_*.pgm")):
+        image, _metadata = sr.read_image(str(path))
+        if image.ndim != 2 or image.dtype != np.uint8:
+            raise RuntimeError(f"{path} did not decode as Gray8")
+        frames.append(image)
+    if len(frames) < 2:
+        raise RuntimeError("need at least two generated frames")
+    return frames
+
+
+def detect_objects(image: np.ndarray):
+    foreground = image >= 100
+    visited = np.zeros_like(foreground, dtype=bool)
+    boxes = []
+    classes = []
+    for y0, x0 in np.argwhere(foreground):
+        if visited[y0, x0]:
+            continue
+        visited[y0, x0] = True
+        queue = deque([(int(x0), int(y0))])
+        xs = []
+        ys = []
+        maximum = 0
+        while queue:
+            x, y = queue.popleft()
+            xs.append(x)
+            ys.append(y)
+            maximum = max(maximum, int(image[y, x]))
+            for nx, ny in ((x - 1, y), (x + 1, y), (x, y - 1), (x, y + 1)):
+                if (
+                    0 <= nx < image.shape[1]
+                    and 0 <= ny < image.shape[0]
+                    and foreground[ny, nx]
+                    and not visited[ny, nx]
+                ):
+                    visited[ny, nx] = True
+                    queue.append((nx, ny))
+        if len(xs) >= 20:
+            boxes.append([min(xs), min(ys), max(xs) + 1, max(ys) + 1])
+            classes.append(2 if maximum >= 210 else 1)
+    order = np.argsort(classes)
+    return (
+        np.asarray(boxes, dtype=np.float32)[order],
+        np.ones(len(boxes), dtype=np.float32)[order],
+        np.asarray(classes, dtype=np.int64)[order],
+    )
+
+
+def render_gif(frames, tracks_per_frame, flows, output: Path) -> None:
+    from PIL import Image, ImageDraw
+
+    colors = {1: (255, 91, 91), 2: (87, 211, 176)}
+    rendered = []
+    for index, (frame, tracks) in enumerate(zip(frames, tracks_per_frame)):
+        canvas = Image.fromarray(frame, mode="L").convert("RGB").resize((WIDTH * 4, HEIGHT * 4))
+        draw = ImageDraw.Draw(canvas)
+        for row in tracks:
+            track_id = int(row[0])
+            box = [int(round(value * 4)) for value in row[1:5]]
+            class_id = int(row[5])
+            color = colors[class_id]
+            draw.rectangle(box, outline=color, width=3)
+            draw.text((box[0] + 3, box[1] + 3), f"id {track_id}", fill=color)
+        if index > 0:
+            flow = flows[index - 1]
+            for y in range(8, HEIGHT - 8, 12):
+                for x in range(8, WIDTH - 8, 12):
+                    dx, dy = flow[y, x]
+                    if np.isfinite(dx) and np.isfinite(dy) and abs(dx) + abs(dy) >= 1:
+                        draw.line(
+                            (x * 4, y * 4, (x + float(dx) * 2) * 4, (y + float(dy) * 2) * 4),
+                            fill=(255, 207, 112),
+                            width=2,
+                        )
+        draw.text((8, 8), f"frame {index:02}", fill=(240, 245, 255))
+        rendered.append(canvas)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    rendered[0].save(
+        output,
+        save_all=True,
+        append_images=rendered[1:],
+        duration=140,
+        loop=0,
+        optimize=False,
+    )
+
+
+def run(frames_dir: Path, gif_path: Path, render: bool = True) -> None:
+    generate_frames(frames_dir)
+    frames = load_frames(frames_dir)
+    tracker = sr.MultiObjectTracker(iou_threshold=0.2, max_missed=1, min_confirmed_hits=2)
+    tracks_per_frame = []
+    flows = []
+    boxes, scores, classes = detect_objects(frames[0])
+    tracks_per_frame.append(tracker.update(boxes, scores, classes))
+    for previous, current in zip(frames, frames[1:]):
+        flow = sr.dense_flow_image(previous, current, block_radius=1, search_radius=3)
+        previous_boxes, _previous_scores, previous_classes = detect_objects(previous)
+        observed = []
+        for box, class_id in zip(previous_boxes, previous_classes):
+            x = int((box[0] + box[2]) * 0.5)
+            y = int((box[1] + box[3]) * 0.5)
+            observed.append((int(class_id), *flow[y, x].tolist()))
+        if observed != [(1, 2.0, 1.0), (2, -2.0, -1.0)]:
+            raise RuntimeError(f"unexpected object flow: {observed}")
+        flows.append(flow)
+        boxes, scores, classes = detect_objects(current)
+        tracks = tracker.update(boxes, scores, classes)
+        if len(tracks) != 2 or [track[0] for track in tracks] != [1, 2]:
+            raise RuntimeError(f"unstable tracks: {tracks}")
+        tracks_per_frame.append(tracks)
+    if render:
+        render_gif(frames, tracks_per_frame, flows, gif_path)
+    print(
+        f"video_tracking_e2e=ok frames={len(frames)} "
+        f"flow_pairs={len(flows)} stable_track_ids=1,2 "
+        f"gif={gif_path if render else 'skipped'}"
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--frames-dir", type=Path, default=Path("target/video-tracking-demo/frames")
+    )
+    parser.add_argument(
+        "--gif", type=Path, default=Path("docs/assets/video_tracking_e2e.gif")
+    )
+    parser.add_argument("--no-gif", action="store_true")
+    args = parser.parse_args()
+    run(args.frames_dir, args.gif, render=not args.no_gif)
+
+
+if __name__ == "__main__":
+    main()

crates/spatialrust-py/spatialrust.pyi

@@ -15,7 +15,7 @@ __all__: list[str] = [
     "__version__", "ImageMetadata", "Tensor", "Keypoint2", "OnnxRuntimeSession",
     "DLPackTensorView", "PointCloud", "PipelineResult", "RegionResult",
     "DbscanResult", "GroundResult", "MultiPlaneResult", "SphereResult",
-    "CylinderResult", "RegistrationResult", "read_image",
+    "CylinderResult", "RegistrationResult", "MultiObjectTracker", "read_image",
     "tensor_copy_from_numpy", "tensor_view_from_dlpack", "harris_keypoints",
     "shi_tomasi_keypoints", "fast_keypoints", "orb_features",
     "estimate_homography_ransac", "solve_pnp", "estimate_rgbd_odometry", "stereo_block_match",
@@ -140,6 +140,27 @@ def dense_flow_image(
     search_radius: int = ...,
 ) -> _F32Array: ...
 
+@final
+class MultiObjectTracker:
+    """Deterministic same-class IoU tracker with persistent track IDs."""
+
+    def __new__(
+        cls,
+        iou_threshold: float = ...,
+        max_missed: int = ...,
+        min_confirmed_hits: int = ...,
+    ) -> MultiObjectTracker: ...
+    def update(
+        self,
+        boxes: _F32Array,
+        scores: _F32Array,
+        class_ids: NDArray[np.int64],
+    ) -> list[
+        tuple[int, float, float, float, float, int, float, int, int, int, bool]
+    ]:
+        """Return ``id, box, class, score, age, hits, missed, confirmed`` tuples."""
+        ...
+
 def rgbd_to_point_cloud(
     depth: _F32Array,
     color: _U8Array,

crates/spatialrust-py/src/lib.rs

@@ -112,9 +112,12 @@ use spatialrust::vision::{
     ObjectImageCorrespondence, OrbOptions, OrbScoreType, PanoramaOptions, PerspectiveTransform,
     PointCorrespondence2, PointMap, RectMorphologyWorkspace, RgbdOdometryOptions, RleOrder,
     RobustEstimationOptions, ShiTomasiOptions, SoftNmsMethod, StereoBmOptions, StructuringElement,
-    ThresholdType,
+    ThresholdType, TrackState,
+};
+use spatialrust::vision::{
+    dense_flow_block_match as dense_flow_native, DenseFlowOptions,
+    MultiObjectTracker as NativeMultiObjectTracker, MultiObjectTrackerOptions,
 };
-use spatialrust::vision::{dense_flow_block_match as dense_flow_native, DenseFlowOptions};
 use spatialrust::voxelize::{
     range_image as range_image_proj, voxelize as voxelize_grid, RangeImageConfig, VoxelFill,
     VoxelGridConfig,
@@ -134,6 +137,7 @@ use spatialrust::{
 
 type Vec3Tuple = (f32, f32, f32);
 type OrientedBoundingBoxTuple = (Vec3Tuple, Vec3Tuple, Vec<Vec3Tuple>);
+type ObjectTrackTuple = (u64, f32, f32, f32, f32, i64, f32, u32, u32, u32, bool);
 type ComponentStats = Vec<(u32, usize, (f32, f32, f32, f32))>;
 
 fn to_py_err<E: std::fmt::Display>(err: E) -> PyErr {
@@ -824,6 +828,81 @@ fn dense_flow_image<'py>(
     Ok(array.into_pyarray_bound(py))
 }
 
+/// Stateful deterministic same-class IoU tracker.
+#[pyclass(name = "MultiObjectTracker")]
+struct PyMultiObjectTracker {
+    inner: NativeMultiObjectTracker,
+}
+
+#[pymethods]
+impl PyMultiObjectTracker {
+    /// Creates an empty tracker.
+    #[new]
+    #[pyo3(signature = (iou_threshold=0.3, max_missed=3, min_confirmed_hits=2))]
+    fn new(iou_threshold: f32, max_missed: u32, min_confirmed_hits: u32) -> PyResult<Self> {
+        let inner = NativeMultiObjectTracker::try_new(MultiObjectTrackerOptions {
+            iou_threshold,
+            max_missed,
+            min_confirmed_hits,
+        })
+        .map_err(to_py_err)?;
+        Ok(Self { inner })
+    }
+
+    /// Associates detections and returns rows ending in a confirmed 0/1 field.
+    fn update(
+        &mut self,
+        boxes: PyReadonlyArray2<'_, f32>,
+        scores: PyReadonlyArray1<'_, f32>,
+        class_ids: PyReadonlyArray1<'_, i64>,
+    ) -> PyResult<Vec<ObjectTrackTuple>> {
+        let boxes = boxes.as_array();
+        let scores = scores.as_array();
+        let class_ids = class_ids.as_array();
+        if boxes.ndim() != 2
+            || boxes.shape()[1] != 4
+            || scores.len() != boxes.shape()[0]
+            || class_ids.len() != boxes.shape()[0]
+        {
+            return Err(PyValueError::new_err(
+                "boxes must be Nx4 with matching scores and class_ids",
+            ));
+        }
+        let detections = boxes
+            .outer_iter()
+            .zip(scores.iter())
+            .zip(class_ids.iter())
+            .map(|((row, &score), &class_id)| {
+                Ok(Detection {
+                    bbox: BoundingBox2::try_new(row[0], row[1], row[2], row[3])
+                        .map_err(to_py_err)?,
+                    score,
+                    class_id,
+                })
+            })
+            .collect::<PyResult<Vec<_>>>()?;
+        let tracks = self.inner.update(&detections).map_err(to_py_err)?;
+        Ok(tracks
+            .iter()
+            .map(|track| {
+                (
+                    track.id,
+                    track.bbox.x_min,
+                    track.bbox.y_min,
+                    track.bbox.x_max,
+                    track.bbox.y_max,
+                    track.class_id,
+                    track.score,
+                    track.age,
+                    track.hits,
+                    track.missed,
+                    track.state == TrackState::Confirmed,
+                )
+            })
+            .collect())
+    }
+}
+
 /// Applies gray-world white balance to an RGB image.
 #[pyfunction]
 fn gray_world_white_balance_image<'py>(
@@ -4162,6 +4241,7 @@ fn spatialrust_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyDistanceTransformWorkspace>()?;
     m.add_class::<PyMorphologyWorkspace>()?;
     m.add_class::<PyCannyWorkspace>()?;
+    m.add_class::<PyMultiObjectTracker>()?;
     m.add_class::<PyOnnxRuntimeSession>()?;
     m.add_class::<PyDlpackTensorView>()?;
     m.add_class::<PyPointCloud>()?;

crates/spatialrust-py/tests/test_bindings.py

@@ -1036,3 +1036,19 @@ def test_dense_flow_binding_recovers_translation_and_marks_border_invalid():
     assert flow.shape == (height, width, 2)
     np.testing.assert_array_equal(flow[16, 20], [2.0, 1.0])
     assert np.isnan(flow[0, 0]).all()
+
+
+def test_multi_object_tracker_preserves_ids_and_confirms():
+    tracker = sr.MultiObjectTracker(iou_threshold=0.2, max_missed=1, min_confirmed_hits=2)
+    scores = np.array([0.9], dtype=np.float32)
+    classes = np.array([3], dtype=np.int64)
+    first = tracker.update(
+        np.array([[10.0, 10.0, 24.0, 22.0]], dtype=np.float32), scores, classes
+    )
+    second = tracker.update(
+        np.array([[12.0, 11.0, 26.0, 23.0]], dtype=np.float32), scores, classes
+    )
+    assert len(first) == len(second) == 1
+    assert first[0][0] == second[0][0] == 1
+    assert first[0][10] is False
+    assert second[0][10] is True