diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d63331..c55af98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,11 @@ removed no sooner than the next major (see `docs/API_STABILITY.md`). ### Added +- **Epic 148C/148D point-cloud conformance aggregator**: `aggregate.py` unifies + PCL/PDAL/Open3D receipts into one `pointcloud-conformance-aggregate` report + with fail-closed checks (unsupported suites, missing ids, duplicate + workloads). The report contract tests grew to eight cases, and the dated + PCL receipt is committed. - **Epic 148A/148B point-cloud conformance harness**: a versioned point-cloud benchmark manifest (`spatialrust.pointcloud-benchmark-manifest.v1`) and a stdlib-only report contract (`spatialrust.pointcloud-comparison.v1`) with diff --git a/bench/pcl_comparison/aggregate.py b/bench/pcl_comparison/aggregate.py new file mode 100644 index 0000000..782aff2 --- /dev/null +++ b/bench/pcl_comparison/aggregate.py @@ -0,0 +1,116 @@ +"""Aggregate and validate point-cloud comparison receipts. + +Collects PCL/PDAL/Open3D comparison receipts, verifies each against the +`spatialrust.pointcloud-comparison.v1` contract, rejects duplicate or +conflicting workloads, and writes one aggregate report. Stdlib-only so it can +gate CI without installing any comparison library. + +Usage: + python bench/pcl_comparison/aggregate.py \ + --receipts bench/pcl_comparison/receipt-*.json \ + [--output target/pointcloud-aggregate.json] +""" + +from __future__ import annotations + +import argparse +import glob +import json +from pathlib import Path + +from report import emit_report, load_report, make_report, validate_report + +SUPPORTED_SUITES = {"pcl_comparison", "pdal_comparison", "open3d_comparison"} + + +def collect(receipt_paths: list[Path]) -> list[dict[str, object]]: + reports = [] + for path in receipt_paths: + reports.append(load_report(path)) + return reports + + +def aggregate(reports: list[dict[str, object]]) -> dict[str, object]: + if not reports: + raise ValueError("at least one receipt is required") + + seen: dict[str, dict[str, object]] = {} + for report in reports: + suite = report["suite"] + if suite not in SUPPORTED_SUITES: + raise ValueError(f"unsupported suite {suite}") + operations = report.get("results", {}).get("operations", []) + if not isinstance(operations, list): + raise ValueError(f"suite {suite} results.operations must be a list") + for operation in operations: + operation_id = operation.get("id") + if not isinstance(operation_id, str) or not operation_id: + raise ValueError(f"suite {suite} has an operation without an id") + if operation_id in seen: + raise ValueError(f"duplicate workload {operation_id} in {suite}") + seen[operation_id] = { + "id": operation_id, + "suite": suite, + "spatialrust_seconds": operation.get("spatialrust_seconds"), + "library_seconds": operation.get( + {"pcl_comparison": "pcl_seconds", "pdal_comparison": "pdal_seconds", "open3d_comparison": "open3d_seconds"}[suite] + ), + "speedup": operation.get("speedup"), + "output_points_sr": operation.get("output_points_sr"), + "output_points_library": operation.get( + {"pcl_comparison": "output_points_pcl", "pdal_comparison": "output_points_pdal", "open3d_comparison": "output_points_open3d"}[suite] + ), + } + + environment = reports[0]["environment"] + return make_report( + suite="pointcloud-conformance-aggregate", + kind="aggregate", + status="pass", + environment_receipt=environment, + results={ + "suite_count": len(reports), + "workload_count": len(seen), + "workloads": list(seen.values()), + }, + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--receipts", + nargs="+", + help="one or more receipt JSON paths or glob patterns", + ) + parser.add_argument("--output", type=Path) + return parser.parse_args() + + +def expand(pattern: str) -> list[Path]: + paths = [Path(match) for match in glob.glob(pattern)] + return [path for path in paths if path.exists()] + + +def main() -> None: + args = parse_args() + patterns = args.receipts or [ + "bench/pcl_comparison/receipt-*.json", + "bench/pdal_comparison/receipt-*.json", + "bench/open3d_comparison/receipt-*.json", + ] + receipt_paths: list[Path] = [] + for pattern in patterns: + receipt_paths.extend(expand(pattern)) + receipt_paths = list(dict.fromkeys(receipt_paths)) + receipt_paths.sort() + if not receipt_paths: + raise SystemExit("no receipt files found") + + reports = collect(receipt_paths) + result = aggregate(reports) + emit_report(result, output=args.output) + + +if __name__ == "__main__": + main() diff --git a/bench/pcl_comparison/test_report.py b/bench/pcl_comparison/test_report.py index 182eebb..ad3e1c2 100755 --- a/bench/pcl_comparison/test_report.py +++ b/bench/pcl_comparison/test_report.py @@ -132,6 +132,61 @@ def test_timing_statistics_and_percentile(): pass +def test_aggregate_merges_suites_and_rejects_duplicates(): + from aggregate import aggregate, collect # noqa: PLC0415 + + base = environment( + pcl_version="1.15.1", + pdal_version="2.6", + open3d_version="0.19.0", + spatialrust_version="1.2.0", + ) + + def suite_report(name, ops): + return make_report( + suite=name, + kind="performance", + status="pass", + environment_receipt=base, + results={"operations": ops}, + ) + + pcl = suite_report( + "pcl_comparison", + [{"id": "voxel_downsample", "spatialrust_seconds": 0.0104, "pcl_seconds": 0.0177}], + ) + pdal = suite_report( + "pdal_comparison", + [{"id": "translate_xyz", "spatialrust_seconds": 0.009, "pdal_seconds": 0.02}], + ) + result = aggregate([pcl, pdal]) + assert result["kind"] == "aggregate" + assert result["status"] == "pass" + assert result["results"]["suite_count"] == 2 + assert result["results"]["workload_count"] == 2 + ids = {w["id"] for w in result["results"]["workloads"]} + assert ids == {"voxel_downsample", "translate_xyz"} + + duplicate = aggregate([pcl, pdal]) + assert duplicate["results"]["suite_count"] == 2 + try: + # A second suite repeating the same workload id must fail closed. + pcl2 = suite_report( + "pdal_comparison", + [{"id": "voxel_downsample", "spatialrust_seconds": 0.0, "pdal_seconds": 0.0}], + ) + aggregate([pcl, pcl2]) + raise AssertionError("duplicate workload must raise") + except ValueError: + pass + try: + # Aggregating the same receipt twice must also fail closed. + aggregate([pcl, pcl]) + raise AssertionError("duplicate suite must raise") + except ValueError: + pass + + def main() -> int: tests = [ ("schema_version_is_canonical", test_schema_version_is_canonical), @@ -141,6 +196,7 @@ def main() -> int: ("wrong_schema_version_fails", test_wrong_schema_version_fails), ("non_finite_values_fail", test_non_finite_values_fail), ("timing_statistics_and_percentile", test_timing_statistics_and_percentile), + ("aggregate_merges_suites_and_rejects_duplicates", test_aggregate_merges_suites_and_rejects_duplicates), ] failures = 0 for name, test in tests: diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 378f146..e79d32e 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -109,8 +109,8 @@ comparison tooling only and never enters a production feature. | --- | --- | --- | --- | | 148A | Complete | Versioned point-cloud benchmark manifest (profiles, statistics, workloads) and stdlib-only report contract | `bench/pcl_comparison/manifest.json`, `report.py`, `test_report.py` | | 148B | Complete | PDAL runner with matching filters/operations on the identical cloud | `bench/pdal_comparison/` | -| 148C | Planned | Unify PCL/PDAL/Open3D comparison receipts and aggregate runner with fail-closed checks | aggregate command | -| 148D | Planned | Dated honest comparison receipt, docs, and README updates | note + `docs/` | +| 148C | Complete | Unify PCL/PDAL/Open3D comparison receipts and aggregate runner with fail-closed checks | `bench/pcl_comparison/aggregate.py` + tests | +| 148D | Complete | Dated honest comparison receipt, docs, and README updates | `receipt-2026-08-07.json`, note | Each slice lands as one reviewable PR. The manifest reserves VGA-class and full-size cloud profiles and at least the operations both libraries implement diff --git a/notes/2026-08-07_epic148_pointcloud_conformance.md b/notes/2026-08-07_epic148_pointcloud_conformance.md index f02aefd..a184d90 100644 --- a/notes/2026-08-07_epic148_pointcloud_conformance.md +++ b/notes/2026-08-07_epic148_pointcloud_conformance.md @@ -1,6 +1,6 @@ # Epic 148: point-cloud conformance program (PCL/PDAL comparison) -Date: 2026-08-07. Slices 148A/148B. +Date: 2026-08-07. Slices 148A–148D complete. ## Why @@ -27,6 +27,10 @@ same fail-closed report contract style. filters.transformation translate, filters.reprojection) and `run.sh` that builds `bench_ops` and prints the side-by-side table. PDAL is comparison tooling only. +- `bench/pcl_comparison/aggregate.py` — unifies PCL/PDAL/Open3D receipts into + one `pointcloud-conformance-aggregate` report. Rejects unsupported suites, + missing operation ids, and duplicate workloads (fail-closed), and validates + every input with `load_report`. - `crates/spatialrust/examples/bench_ops.rs` — adds a `translate_xyz` workload behind `transform-ops` so SpatialRust can be compared against PDAL's transform filter. @@ -47,7 +51,8 @@ numbers, not portability guarantees. ## Next slices -148C unifies PCL/PDAL/Open3D receipts into an aggregate runner with fail-closed -checks; 148D publishes docs and a dated receipt. PDAL/Open3D were not installed -on this host, so their runners are ready but their dated numbers must be -produced on a machine with those tools. +Epic 148 is complete. A future `full` (2M-point synthetic room) dated run and +PDAL/Open3D dated receipts can be produced on machines with those tools; the +aggregate runner already validates all three suite kinds. PDAL/Open3D were not +installed on this host, so their runners are ready but their dated numbers must +be produced on a machine with those tools.