[0.4.5] Strengthen OSW coordinate and geometry validation

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Change log
 
+### 0.4.5 - 2026-07-21
+- Added immutable `ValidationConfig` support. Users can override the 2,000-vertex limit, seven-decimal coordinate warning threshold, and zero-length line handling per validator instance. Zero-length `LineString` geometries are rejected by default and can be allowed with `allow_zero_length_lines=True`.
+- Fixed [#3982](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/3982): edge `_u_id`/`_v_id` endpoints and zone `_w_id` vertices must now exactly match their referenced node coordinates; the previous `1e-7` tolerance was removed.
+- Added [#4047](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/4047): input coordinates with more than seven decimal places produce a non-blocking message through `ValidationResult.warnings` without changing validation errors or validity.
+- Added [#4048](https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/4048): edge, line, polygon, and zone geometries containing more than 2,000 vertices are rejected, including supported external line and polygon geometries.
+- Added feature-level errors for line and polygon geometries whose coordinates all collapse to one location, including the feature ID, dataset, and feature index, while suppressing duplicate generic invalid-geometry messages.
+- Added boundary, regression, and real-data coverage for exact coordinate matching, coordinate precision warnings, geometry vertex limits, and collapsed-geometry reporting.
+
 ### 0.4.4 - 2026-07-01
 - Fixed dataset filename classification so names containing dataset tokens in the prefix, such as `River._edges_.nodes.geojson`, are classified by their final dataset suffix instead of by loose substring matching.
 - Supported filename detection now consistently accepts only these four forms for each dataset type: `<dataset>.geojson`, `*.<dataset>.geojson`, `<dataset>.OSW.geojson`, and `*.<dataset>.OSW.geojson`.

README.md

@@ -39,13 +39,45 @@ result = validator.validate()
 print(result.is_valid)
 print(result.errors)  # returns up to the first 20 high-level errors by default
 print(result.issues)  # detailed per-feature issues, capped to first 20 by default
+print(result.warnings)  # non-blocking coordinate precision warning, or an empty string
 
 result = validator.validate(max_errors=10)
 print(result.is_valid)
 print(result.errors)  # returns up to the first 10 high-level errors
 print(result.issues)  # capped by the same max_errors limit
 ```
 
+### Validation configuration
+
+Use `ValidationConfig` to override validation behavior for one validator instance.
+When no configuration is supplied, the documented defaults are used.
+
+```python
+from python_osw_validation import OSWValidation, ValidationConfig
+
+config = ValidationConfig(
+    max_geometry_vertices=3000,
+    coordinate_precision=8,
+    allow_zero_length_lines=True,
+)
+
+validator = OSWValidation(
+    zipfile_path="dataset.zip",
+    config=config,
+)
+result = validator.validate()
+```
+
+| Setting | Default | Meaning |
+|---------|---------|---------|
+| `max_geometry_vertices` | `2000` | Maximum allowed vertices for edges, lines, polygons, and zones. Must be an integer greater than zero. |
+| `coordinate_precision` | `7` | Maximum coordinate decimal places before a non-blocking warning is reported. Must be a non-negative integer. |
+| `allow_zero_length_lines` | `False` | Rejects collapsed `LineString` geometries by default. Set to `True` to allow them in edges, lines, and external line data. |
+
+Allowing zero-length lines does not bypass `_u_id`/`_v_id` existence or exact
+node-coordinate checks. Zero-area polygons and zones, collapsed `MultiLineString`
+geometries, and line geometries in point datasets remain invalid.
+
 ## Error behavior
 
 - `errors`: high-level validation messages, capped by `max_errors` (default `20`).

src/example.py

@@ -1,10 +1,10 @@
 import os
-from python_osw_validation import OSWValidation
+from python_osw_validation import OSWValidation, ValidationConfig
 
 PARENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 ASSETS_DIR = os.path.join(PARENT_DIR, 'tests/assets')
 # VALID_ZIP_FILE = os.path.join(ASSETS_DIR, 'valid 2.zip')
-VALID_ZIP_FILE = os.path.join(ASSETS_DIR, '33493.zip')
+VALID_ZIP_FILE = os.path.join(ASSETS_DIR, 'issue_2_sanitize.zip')
 INVALID_ZIP_FILE = os.path.join(ASSETS_DIR, '4151.zip')
 INVALID_VANCOUVER_ZIP_FILE = os.path.join(ASSETS_DIR, 'vancouver-dataset.zip')
 SCHEMA_DIR = os.path.join(PARENT_DIR, 'src/python_osw_validation/schema')
@@ -18,9 +18,15 @@ def valid_test_with_provided_schema():
 
 
 def valid_test_without_provided_schema():
-    validator = OSWValidation(zipfile_path=VALID_ZIP_FILE)
+    config = ValidationConfig(
+        max_geometry_vertices=1,
+        coordinate_precision=7,
+        allow_zero_length_lines=True,
+    )
+    validator = OSWValidation(zipfile_path=VALID_ZIP_FILE, config=config)
     result = validator.validate()
     print(result.issues)
+    print(result.warnings)
     # print(f'Valid Test Without Schema: {"Passed" if result.is_valid else "Failed"}')
 
 

src/python_osw_validation/__init__.py

@@ -8,6 +8,11 @@
 import jsonschema_rs
 
 from .zipfile_handler import ZipFileHandler
+from .config import (
+    DEFAULT_COORDINATE_PRECISION,
+    DEFAULT_MAX_GEOMETRY_VERTICES,
+    ValidationConfig,
+)
 from .extracted_data_validator import ExtractedDataValidator, OSW_DATASET_FILES, dataset_key_for_filename
 from .version import __version__
 from .helpers import (
@@ -16,6 +21,7 @@
     _feature_index_from_error,
     _pretty_message,
     _read_geojson_without_ext,
+    _geojson_file_has_excess_coordinate_precision,
 )
 
 SCHEMA_PATH = os.path.join(os.path.dirname(__file__), 'schema')
@@ -28,23 +34,32 @@
     "zones": os.path.join(SCHEMA_PATH, 'opensidewalks.zones.schema-0.3.json'),
 }
 
+COORDINATE_PRECISION_WARNING = (
+    "Input dataset contains coordinates with more than "
+    f"{DEFAULT_COORDINATE_PRECISION} decimal places."
+)
+MAX_GEOMETRY_VERTICES = DEFAULT_MAX_GEOMETRY_VERTICES
+VERTEX_LIMIT_DATASETS = frozenset({"edges", "lines", "polygons", "zones"})
+
 
 class ValidationResult:
     """Container for validation outcome.
 
     * `errors`: high-level, human-readable strings (legacy behavior).
     * `issues`: per-feature schema problems (former `fixme`), each item:
         { 'filename': str, 'feature_index': Optional[int], 'error_message': List[str] }
+    * `warnings`: non-blocking validation warning string.
     """
 
     def __init__(self, is_valid: bool, errors: Optional[List[str]] = None,
-                 issues: Optional[List[Dict[str, Any]]] = None):
+                 issues: Optional[List[Dict[str, Any]]] = None, warnings: str = ""):
         self.is_valid = is_valid
         if len(errors) == 0:
             self.errors = None
         else:
             self.errors = errors
         self.issues = issues
+        self.warnings = warnings
 
 
 class OSWValidation:
@@ -63,12 +78,17 @@ def __init__(
             point_schema_path: Optional[str] = None,
             line_schema_path: Optional[str] = None,
             polygon_schema_path: Optional[str] = None,
+            config: Optional[ValidationConfig] = None,
     ):
         self.zipfile_path = zipfile_path
         self.extracted_dir: Optional[str] = None
         self.errors: List[str] = []
         # per-feature schema issues (formerly `fixme`)
         self.issues: List[Dict[str, Any]] = []
+        self.warnings = ""
+        if config is not None and not isinstance(config, ValidationConfig):
+            raise TypeError("config must be a ValidationConfig instance.")
+        self.config = config or ValidationConfig()
 
         # Legacy single schema (if set, used for all)
         self.schema_file_path = schema_file_path  # may be None
@@ -95,6 +115,23 @@ def log_errors(self, message: str, filename: Optional[str] = None, feature_index
             'error_message': message,
         })
 
+    def _check_coordinate_precision(self, file_paths) -> None:
+        """Set one aggregate warning without affecting validation state."""
+        for file_path in file_paths:
+            try:
+                if _geojson_file_has_excess_coordinate_precision(
+                    str(file_path),
+                    self.config.coordinate_precision,
+                ):
+                    self.warnings = (
+                        "Input dataset contains coordinates with more than "
+                        f"{self.config.coordinate_precision} decimal places."
+                    )
+                    return
+            except (OSError, json.JSONDecodeError, TypeError, ValueError):
+                # Existing parsing and schema validation own all functional errors.
+                continue
+
     # add this small helper inside OSWValidation (near other helpers)
     def _get_colset(self, gdf: Optional[gpd.GeoDataFrame], col: str, filekey: str) -> set:
         """Return set of a column if present; else log and return empty set."""
@@ -117,10 +154,8 @@ def _get_colset(self, gdf: Optional[gpd.GeoDataFrame], col: str, filekey: str) -
     # Geometry mapping helpers
     # ----------------------------
 
-    _COORD_TOLERANCE = 1e-7  # ~1 cm at equator
-
     def _coords_match(self, c1: tuple, c2: tuple) -> bool:
-        return abs(c1[0] - c2[0]) <= self._COORD_TOLERANCE and abs(c1[1] - c2[1]) <= self._COORD_TOLERANCE
+        return c1 == c2
 
     def _build_node_coord_map(self, nodes_df: gpd.GeoDataFrame) -> Dict[Any, tuple]:
         """Return {node_id: (lon, lat)} from a nodes GeoDataFrame."""
@@ -263,6 +298,125 @@ def _validate_zone_geometry_mapping(
                         feature_index=feat_idx,
                     )
 
+    @staticmethod
+    def _ring_vertex_count(ring) -> int:
+        coords = list(ring.coords)
+        if len(coords) > 1 and coords[0] == coords[-1]:
+            return len(coords) - 1
+        return len(coords)
+
+    def _geometry_vertex_count(self, geom) -> int:
+        """Count line and polygon vertices, excluding polygon ring closure coordinates."""
+        if geom is None:
+            return 0
+        if geom.geom_type == 'LineString':
+            return len(geom.coords)
+        if geom.geom_type == 'Polygon':
+            return self._ring_vertex_count(geom.exterior) + sum(
+                self._ring_vertex_count(ring) for ring in geom.interiors
+            )
+        if geom.geom_type in {'MultiLineString', 'MultiPolygon', 'GeometryCollection'}:
+            return sum(self._geometry_vertex_count(part) for part in geom.geoms)
+        return 0
+
+    def _validate_geometry_vertex_limit(
+        self,
+        gdf: Optional[gpd.GeoDataFrame],
+        dataset_name: str,
+        max_errors: int,
+    ) -> None:
+        """Report line or polygon features exceeding the configured vertex limit."""
+        if gdf is None:
+            return
+
+        for feature_index, row in gdf.iterrows():
+            if len(self.errors) >= max_errors:
+                break
+
+            vertex_count = self._geometry_vertex_count(row.geometry)
+            if vertex_count <= self.config.max_geometry_vertices:
+                continue
+
+            feature_id = row.get('_id', feature_index)
+            if self._is_nullish_value(feature_id):
+                feature_id = feature_index
+            self.log_errors(
+                message=(
+                    f"Feature '{feature_id}' in '{dataset_name}' contains {vertex_count} geometry vertices. "
+                    f"Maximum allowed is {self.config.max_geometry_vertices}."
+                ),
+                filename=dataset_name,
+                feature_index=feature_index,
+            )
+
+    def _geometry_coordinate_pairs(self, geometry) -> List[tuple]:
+        """Return two-dimensional coordinate pairs from line and polygon geometries."""
+        if geometry is None or geometry.is_empty:
+            return []
+        if geometry.geom_type in {'LineString', 'LinearRing'}:
+            return [(coordinate[0], coordinate[1]) for coordinate in geometry.coords]
+        if geometry.geom_type == 'Polygon':
+            coordinates = self._geometry_coordinate_pairs(geometry.exterior)
+            for ring in geometry.interiors:
+                coordinates.extend(self._geometry_coordinate_pairs(ring))
+            return coordinates
+        if geometry.geom_type in {'MultiLineString', 'MultiPolygon', 'GeometryCollection'}:
+            return [
+                coordinate
+                for part in geometry.geoms
+                for coordinate in self._geometry_coordinate_pairs(part)
+            ]
+        return []
+
+    def _validate_collapsed_geometries(
+        self,
+        gdf: Optional[gpd.GeoDataFrame],
+        dataset_name: str,
+        max_errors: int,
+    ) -> set:
+        """Report line or polygon features whose coordinates all share one location."""
+        collapsed_indexes = set()
+        if gdf is None:
+            return collapsed_indexes
+
+        for feature_index, row in gdf.iterrows():
+            geometry = row.geometry
+            coordinates = self._geometry_coordinate_pairs(geometry)
+            if len(coordinates) < 2 or len(set(coordinates)) != 1:
+                continue
+
+            if geometry.geom_type in {'LineString', 'MultiLineString'}:
+                geometry_problem = 'zero-length'
+            elif geometry.geom_type in {'Polygon', 'MultiPolygon'}:
+                geometry_problem = 'zero-area'
+            else:
+                continue
+
+            collapsed_indexes.add(feature_index)
+            allow_zero_length_line = (
+                self.config.allow_zero_length_lines
+                and geometry.geom_type == 'LineString'
+                and dataset_name not in {"nodes", "points", "polygons", "zones"}
+            )
+            if allow_zero_length_line:
+                continue
+            if len(self.errors) >= max_errors:
+                continue
+
+            feature_id = row.get('_id', feature_index)
+            if self._is_nullish_value(feature_id):
+                feature_id = feature_index
+            self.log_errors(
+                message=(
+                    f"Feature '{feature_id}' in '{dataset_name}' has {geometry_problem} geometry "
+                    "because all coordinates are identical."
+                ),
+                filename=dataset_name,
+                feature_index=feature_index,
+            )
+
+        return collapsed_indexes
+
     def _schema_key_from_text(self, text: Optional[str]) -> Optional[str]:
         """Return dataset key from exact filename suffixes only."""
         if not text:
@@ -394,7 +548,7 @@ def _finalize(is_valid: bool, errors: Optional[List[str]] = None) -> ValidationR
             final_errors = self.errors if errors is None else errors
             final_errors = (final_errors or [])[:max_errors]
             final_issues = (self.issues or [])[:max_errors]
-            return ValidationResult(is_valid, final_errors, final_issues)
+            return ValidationResult(is_valid, final_errors, final_issues, self.warnings)
 
         zip_handler = None
         OSW_DATASET: Dict[str, Optional[gpd.GeoDataFrame]] = {}
@@ -423,6 +577,11 @@ def _finalize(is_valid: bool, errors: Optional[List[str]] = None) -> ValidationR
                 )
                 return _finalize(False)
 
+            self._check_coordinate_precision(
+                list(getattr(validator, 'files', []) or [])
+                + list(getattr(validator, 'externalExtensions', []) or [])
+            )
+
             # Per-file schema validation → populate self.issues (fixme-like)
             for file in validator.files:
                 file_path = os.path.join(file)
@@ -549,6 +708,9 @@ def _finalize(is_valid: bool, errors: Optional[List[str]] = None) -> ValidationR
             for osw_file, gdf in OSW_DATASET.items():
                 if gdf is None:
                     continue
+                collapsed_indexes = self._validate_collapsed_geometries(gdf, osw_file, max_errors)
+                if osw_file in VERTEX_LIMIT_DATASETS and len(self.errors) < max_errors:
+                    self._validate_geometry_vertex_limit(gdf, osw_file, max_errors)
                 expected_geom = OSW_DATASET_FILES.get(osw_file, {}).get('geometry')
                 if expected_geom:
                     invalid_geojson = gdf[
@@ -557,6 +719,9 @@ def _finalize(is_valid: bool, errors: Optional[List[str]] = None) -> ValidationR
                 else:
                     invalid_geojson = gdf[gdf.is_valid == False]
 
+                if collapsed_indexes:
+                    invalid_geojson = invalid_geojson.drop(index=collapsed_indexes, errors='ignore')
+
                 if len(invalid_geojson) > 0:
                     # Extract IDs if present, else fallback to index
                     ids_series = invalid_geojson['_id'] if '_id' in invalid_geojson.columns else invalid_geojson.index
@@ -585,7 +750,18 @@ def _finalize(is_valid: bool, errors: Optional[List[str]] = None) -> ValidationR
                     )
                     continue
 
+                if len(self.errors) < max_errors:
+                    self._validate_geometry_vertex_limit(extensionFile, file_name, max_errors)
+
+                collapsed_indexes = self._validate_collapsed_geometries(
+                    extensionFile,
+                    file_name,
+                    max_errors,
+                )
+
                 invalid_geojson = extensionFile[extensionFile.is_valid == False]
+                if collapsed_indexes:
+                    invalid_geojson = invalid_geojson.drop(index=collapsed_indexes, errors='ignore')
                 if len(invalid_geojson) > 0:
                     try:
                         invalid_ids = list(set(invalid_geojson.get('_id', invalid_geojson.index)))