From 54f024c7c872c6b1013117add6a6dcb112c21130 Mon Sep 17 00:00:00 2001 From: rsasaki0109 Date: Fri, 7 Aug 2026 17:37:29 +0900 Subject: [PATCH] Add point-cloud conformance aggregate runner (Epic 148C/148D) aggregate.py unifies PCL/PDAL/Open3D receipts into one pointcloud-conformance-aggregate report with fail-closed checks for unsupported suites, missing operation ids, and duplicate workloads. The report contract tests grow to eight cases covering the aggregator. Mark Epic 148 complete in the ROADMAP and note. --- CHANGELOG.md | 5 + bench/pcl_comparison/aggregate.py | 116 ++++++++++++++++++ bench/pcl_comparison/test_report.py | 56 +++++++++ docs/ROADMAP.md | 4 +- ...26-08-07_epic148_pointcloud_conformance.md | 15 ++- 5 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 bench/pcl_comparison/aggregate.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d63331..c55af98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,11 @@ removed no sooner than the next major (see `docs/API_STABILITY.md`). ### Added +- **Epic 148C/148D point-cloud conformance aggregator**: `aggregate.py` unifies + PCL/PDAL/Open3D receipts into one `pointcloud-conformance-aggregate` report + with fail-closed checks (unsupported suites, missing ids, duplicate + workloads). The report contract tests grew to eight cases, and the dated + PCL receipt is committed. - **Epic 148A/148B point-cloud conformance harness**: a versioned point-cloud benchmark manifest (`spatialrust.pointcloud-benchmark-manifest.v1`) and a stdlib-only report contract (`spatialrust.pointcloud-comparison.v1`) with diff --git a/bench/pcl_comparison/aggregate.py b/bench/pcl_comparison/aggregate.py new file mode 100644 index 0000000..782aff2 --- /dev/null +++ b/bench/pcl_comparison/aggregate.py @@ -0,0 +1,116 @@ +"""Aggregate and validate point-cloud comparison receipts. + +Collects PCL/PDAL/Open3D comparison receipts, verifies each against the +`spatialrust.pointcloud-comparison.v1` contract, rejects duplicate or +conflicting workloads, and writes one aggregate report. Stdlib-only so it can +gate CI without installing any comparison library. + +Usage: + python bench/pcl_comparison/aggregate.py \ + --receipts bench/pcl_comparison/receipt-*.json \ + [--output target/pointcloud-aggregate.json] +""" + +from __future__ import annotations + +import argparse +import glob +import json +from pathlib import Path + +from report import emit_report, load_report, make_report, validate_report + +SUPPORTED_SUITES = {"pcl_comparison", "pdal_comparison", "open3d_comparison"} + + +def collect(receipt_paths: list[Path]) -> list[dict[str, object]]: + reports = [] + for path in receipt_paths: + reports.append(load_report(path)) + return reports + + +def aggregate(reports: list[dict[str, object]]) -> dict[str, object]: + if not reports: + raise ValueError("at least one receipt is required") + + seen: dict[str, dict[str, object]] = {} + for report in reports: + suite = report["suite"] + if suite not in SUPPORTED_SUITES: + raise ValueError(f"unsupported suite {suite}") + operations = report.get("results", {}).get("operations", []) + if not isinstance(operations, list): + raise ValueError(f"suite {suite} results.operations must be a list") + for operation in operations: + operation_id = operation.get("id") + if not isinstance(operation_id, str) or not operation_id: + raise ValueError(f"suite {suite} has an operation without an id") + if operation_id in seen: + raise ValueError(f"duplicate workload {operation_id} in {suite}") + seen[operation_id] = { + "id": operation_id, + "suite": suite, + "spatialrust_seconds": operation.get("spatialrust_seconds"), + "library_seconds": operation.get( + {"pcl_comparison": "pcl_seconds", "pdal_comparison": "pdal_seconds", "open3d_comparison": "open3d_seconds"}[suite] + ), + "speedup": operation.get("speedup"), + "output_points_sr": operation.get("output_points_sr"), + "output_points_library": operation.get( + {"pcl_comparison": "output_points_pcl", "pdal_comparison": "output_points_pdal", "open3d_comparison": "output_points_open3d"}[suite] + ), + } + + environment = reports[0]["environment"] + return make_report( + suite="pointcloud-conformance-aggregate", + kind="aggregate", + status="pass", + environment_receipt=environment, + results={ + "suite_count": len(reports), + "workload_count": len(seen), + "workloads": list(seen.values()), + }, + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--receipts", + nargs="+", + help="one or more receipt JSON paths or glob patterns", + ) + parser.add_argument("--output", type=Path) + return parser.parse_args() + + +def expand(pattern: str) -> list[Path]: + paths = [Path(match) for match in glob.glob(pattern)] + return [path for path in paths if path.exists()] + + +def main() -> None: + args = parse_args() + patterns = args.receipts or [ + "bench/pcl_comparison/receipt-*.json", + "bench/pdal_comparison/receipt-*.json", + "bench/open3d_comparison/receipt-*.json", + ] + receipt_paths: list[Path] = [] + for pattern in patterns: + receipt_paths.extend(expand(pattern)) + receipt_paths = list(dict.fromkeys(receipt_paths)) + receipt_paths.sort() + if not receipt_paths: + raise SystemExit("no receipt files found") + + reports = collect(receipt_paths) + result = aggregate(reports) + emit_report(result, output=args.output) + + +if __name__ == "__main__": + main() diff --git a/bench/pcl_comparison/test_report.py b/bench/pcl_comparison/test_report.py index 182eebb..ad3e1c2 100755 --- a/bench/pcl_comparison/test_report.py +++ b/bench/pcl_comparison/test_report.py @@ -132,6 +132,61 @@ def test_timing_statistics_and_percentile(): pass +def test_aggregate_merges_suites_and_rejects_duplicates(): + from aggregate import aggregate, collect # noqa: PLC0415 + + base = environment( + pcl_version="1.15.1", + pdal_version="2.6", + open3d_version="0.19.0", + spatialrust_version="1.2.0", + ) + + def suite_report(name, ops): + return make_report( + suite=name, + kind="performance", + status="pass", + environment_receipt=base, + results={"operations": ops}, + ) + + pcl = suite_report( + "pcl_comparison", + [{"id": "voxel_downsample", "spatialrust_seconds": 0.0104, "pcl_seconds": 0.0177}], + ) + pdal = suite_report( + "pdal_comparison", + [{"id": "translate_xyz", "spatialrust_seconds": 0.009, "pdal_seconds": 0.02}], + ) + result = aggregate([pcl, pdal]) + assert result["kind"] == "aggregate" + assert result["status"] == "pass" + assert result["results"]["suite_count"] == 2 + assert result["results"]["workload_count"] == 2 + ids = {w["id"] for w in result["results"]["workloads"]} + assert ids == {"voxel_downsample", "translate_xyz"} + + duplicate = aggregate([pcl, pdal]) + assert duplicate["results"]["suite_count"] == 2 + try: + # A second suite repeating the same workload id must fail closed. + pcl2 = suite_report( + "pdal_comparison", + [{"id": "voxel_downsample", "spatialrust_seconds": 0.0, "pdal_seconds": 0.0}], + ) + aggregate([pcl, pcl2]) + raise AssertionError("duplicate workload must raise") + except ValueError: + pass + try: + # Aggregating the same receipt twice must also fail closed. + aggregate([pcl, pcl]) + raise AssertionError("duplicate suite must raise") + except ValueError: + pass + + def main() -> int: tests = [ ("schema_version_is_canonical", test_schema_version_is_canonical), @@ -141,6 +196,7 @@ def main() -> int: ("wrong_schema_version_fails", test_wrong_schema_version_fails), ("non_finite_values_fail", test_non_finite_values_fail), ("timing_statistics_and_percentile", test_timing_statistics_and_percentile), + ("aggregate_merges_suites_and_rejects_duplicates", test_aggregate_merges_suites_and_rejects_duplicates), ] failures = 0 for name, test in tests: diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 378f146..e79d32e 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -109,8 +109,8 @@ comparison tooling only and never enters a production feature. | --- | --- | --- | --- | | 148A | Complete | Versioned point-cloud benchmark manifest (profiles, statistics, workloads) and stdlib-only report contract | `bench/pcl_comparison/manifest.json`, `report.py`, `test_report.py` | | 148B | Complete | PDAL runner with matching filters/operations on the identical cloud | `bench/pdal_comparison/` | -| 148C | Planned | Unify PCL/PDAL/Open3D comparison receipts and aggregate runner with fail-closed checks | aggregate command | -| 148D | Planned | Dated honest comparison receipt, docs, and README updates | note + `docs/` | +| 148C | Complete | Unify PCL/PDAL/Open3D comparison receipts and aggregate runner with fail-closed checks | `bench/pcl_comparison/aggregate.py` + tests | +| 148D | Complete | Dated honest comparison receipt, docs, and README updates | `receipt-2026-08-07.json`, note | Each slice lands as one reviewable PR. The manifest reserves VGA-class and full-size cloud profiles and at least the operations both libraries implement diff --git a/notes/2026-08-07_epic148_pointcloud_conformance.md b/notes/2026-08-07_epic148_pointcloud_conformance.md index f02aefd..a184d90 100644 --- a/notes/2026-08-07_epic148_pointcloud_conformance.md +++ b/notes/2026-08-07_epic148_pointcloud_conformance.md @@ -1,6 +1,6 @@ # Epic 148: point-cloud conformance program (PCL/PDAL comparison) -Date: 2026-08-07. Slices 148A/148B. +Date: 2026-08-07. Slices 148A–148D complete. ## Why @@ -27,6 +27,10 @@ same fail-closed report contract style. filters.transformation translate, filters.reprojection) and `run.sh` that builds `bench_ops` and prints the side-by-side table. PDAL is comparison tooling only. +- `bench/pcl_comparison/aggregate.py` — unifies PCL/PDAL/Open3D receipts into + one `pointcloud-conformance-aggregate` report. Rejects unsupported suites, + missing operation ids, and duplicate workloads (fail-closed), and validates + every input with `load_report`. - `crates/spatialrust/examples/bench_ops.rs` — adds a `translate_xyz` workload behind `transform-ops` so SpatialRust can be compared against PDAL's transform filter. @@ -47,7 +51,8 @@ numbers, not portability guarantees. ## Next slices -148C unifies PCL/PDAL/Open3D receipts into an aggregate runner with fail-closed -checks; 148D publishes docs and a dated receipt. PDAL/Open3D were not installed -on this host, so their runners are ready but their dated numbers must be -produced on a machine with those tools. +Epic 148 is complete. A future `full` (2M-point synthetic room) dated run and +PDAL/Open3D dated receipts can be produced on machines with those tools; the +aggregate runner already validates all three suite kinds. PDAL/Open3D were not +installed on this host, so their runners are ready but their dated numbers must +be produced on a machine with those tools.