diff --git a/HISTORY.rst b/HISTORY.rst index cf9720f..4cceae4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,16 @@ History ======= +1.1.8 (unreleased) +------------------ + +* Added typed Safe Data Preview support: + + * ``safe_data_preview`` on ``InDataDiscoveryConfig`` (via ``SafeDataPreviewOptions``) to configure or disable it. + * ``safe_data_preview`` on schema-discovery result columns and file-discovery locators, typed by ``kind``. + +* Added ``row_count`` to ``TableConstraints`` in schema-discovery ``table_metadata``. + 1.1.7 (2026-07-14) ------------------ diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index e2275e3..6ec133f 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -116,6 +116,38 @@ RunInfo, UnfinishedRun, ) +from datamasque.client.models.safe_data_preview import ( + BooleanPreview, + BooleanStatistics, + ColumnKind, + ColumnPreview, + CommonStatistics, + FirstCharsStatistics, + LengthEntry, + LengthsStatistics, + MaskedFormEntry, + NumericBin, + NumericHistograms, + NumericPreview, + NumericStatistics, + NumericSummaries, + NumericTemporalDisclosureLevel, + PatternComposition, + PatternEntry, + PatternsStatistics, + SafeDataPreview, + SafeDataPreviewOptions, + StringDisclosureLevel, + StringPreview, + StringStatistics, + TemporalBin, + TemporalHistograms, + TemporalPreview, + TemporalStatistics, + TemporalSummaries, + UnsupportedPreview, + UnsupportedStatistics, +) from datamasque.client.models.status import ( AsyncRulesetGenerationTaskStatus, MaskingRunStatus, @@ -131,6 +163,11 @@ "AsyncRulesetGenerationInProgressError", "AsyncRulesetGenerationTaskStatus", "AzureConnectionConfig", + "BooleanPreview", + "BooleanStatistics", + "ColumnKind", + "ColumnPreview", + "CommonStatistics", "ConnectionConfig", "ConnectionId", "ConstraintColumns", @@ -171,6 +208,7 @@ "FileId", "FileOrContent", "FileRulesetGenerationRequest", + "FirstCharsStatistics", "ForeignKeyRef", "GitSnapshot", "HashColumnsTableConfig", @@ -186,16 +224,28 @@ "InvalidLibraryError", "InvalidRulesetError", "JsonPath", + "LengthEntry", + "LengthsStatistics", "LicenseInfo", "Locator", "MaskType", + "MaskedFormEntry", "MaskingRunOptions", "MaskingRunRequest", "MaskingRunStatus", "MongoConnectionConfig", "MountedShareConnectionConfig", "MssqlLinkedServerConnectionConfig", + "NumericBin", + "NumericHistograms", + "NumericPreview", + "NumericStatistics", + "NumericSummaries", + "NumericTemporalDisclosureLevel", "OracleWalletFile", + "PatternComposition", + "PatternEntry", + "PatternsStatistics", "ReferencingForeignKey", "Ruleset", "RulesetGenerationRequest", @@ -213,6 +263,8 @@ "RunInfo", "RunNotCancellableError", "S3ConnectionConfig", + "SafeDataPreview", + "SafeDataPreviewOptions", "SchemaDiscoveryColumn", "SchemaDiscoveryFromConfigRequest", "SchemaDiscoveryPage", @@ -228,9 +280,19 @@ "SseConfig", "SseSelection", "SslZipFile", + "StringDisclosureLevel", + "StringPreview", + "StringStatistics", "SwitchableLicenseMetadata", "TableConstraints", + "TemporalBin", + "TemporalHistograms", + "TemporalPreview", + "TemporalStatistics", + "TemporalSummaries", "UnfinishedRun", + "UnsupportedPreview", + "UnsupportedStatistics", "User", "UserId", "UserRole", diff --git a/datamasque/client/models/discovery.py b/datamasque/client/models/discovery.py index 23a5a88..7fe8e6a 100644 --- a/datamasque/client/models/discovery.py +++ b/datamasque/client/models/discovery.py @@ -10,6 +10,10 @@ from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, unwrap_discovery_config_id from datamasque.client.models.pagination import Page from datamasque.client.models.runs import RunConnectionRef +from datamasque.client.models.safe_data_preview import ( + SafeDataPreview, + SafeDataPreviewOptions, +) class InDataDiscoveryRule(BaseModel): @@ -32,6 +36,7 @@ class InDataDiscoveryConfig(BaseModel): non_sensitive_rules: Optional[list[InDataDiscoveryRule]] = None ignore_rules: Optional[list[InDataDiscoveryRule]] = None force: Optional[bool] = None + safe_data_preview: Optional[SafeDataPreviewOptions] = None class SchemaDiscoveryRequest(BaseModel): @@ -294,6 +299,7 @@ class SchemaDiscoveryColumn(BaseModel): unique_index_names: list[str] referencing_foreign_keys: list[ReferencingForeignKey] constraint: str # Primary or Unique, or empty string if column does not participate in a PK/UK + safe_data_preview: Optional[SafeDataPreview] = None class SchemaDiscoveryResult(BaseModel): @@ -324,6 +330,7 @@ class TableConstraints(BaseModel): primary_keys: Optional[list[ConstraintColumns]] = None unique_keys: Optional[list[ConstraintColumns]] = None foreign_keys: Optional[list[ConstraintColumns]] = None + row_count: Optional[int] = None class SchemaDiscoveryPage(Page[SchemaDiscoveryResult]): @@ -356,6 +363,7 @@ class FileDiscoveryLocatorResult(BaseModel): locator: Locator matches: list[FileDiscoveryMatch] data_types: list[str] + safe_data_preview: Optional[SafeDataPreview] = None class FileDiscoveryFile(BaseModel): diff --git a/datamasque/client/models/safe_data_preview.py b/datamasque/client/models/safe_data_preview.py new file mode 100644 index 0000000..258f0a2 --- /dev/null +++ b/datamasque/client/models/safe_data_preview.py @@ -0,0 +1,282 @@ +"""Typed request and response shapes for Safe Data Preview, part of In-Data Discovery.""" + +from enum import Enum +from typing import Annotated, Literal, Optional, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, +) + + +class StringDisclosureLevel(Enum): + """How much detail a string Safe Data Preview reveals.""" + + lengths = "lengths" + patterns = "patterns" + first_chars = "first_chars" + + +class NumericTemporalDisclosureLevel(Enum): + """How much detail a numeric or temporal Safe Data Preview reveals.""" + + summaries = "summaries" + histograms = "histograms" + + +class SafeDataPreviewOptions(BaseModel): + """Configures whether Safe Data Preview runs and how much detail each column reveals.""" + + model_config = ConfigDict(extra="forbid") + + enabled: Optional[bool] = None + min_group_size: Optional[int] = None + string_level: Optional[StringDisclosureLevel] = None + numeric_level: Optional[NumericTemporalDisclosureLevel] = None + temporal_level: Optional[NumericTemporalDisclosureLevel] = None + + +class ColumnKind(str, Enum): + """Which shape a column's Safe Data Preview takes.""" + + boolean = "boolean" + numeric = "numeric" + string = "string" + temporal = "temporal" + unsupported = "unsupported" + + +class CommonStatistics(BaseModel): + """Row counts shared by every preview kind.""" + + model_config = ConfigDict(extra="allow") + + count_row: int + count_null: int + count_distinct: Optional[int] = None + + +class LengthEntry(BaseModel): + """A value length and how many sampled rows had it.""" + + model_config = ConfigDict(extra="allow") + + length: int + count: int + + +class LengthsStatistics(BaseModel): + """Distribution of value lengths.""" + + model_config = ConfigDict(extra="allow") + + min: int + max: int + mean: float + median: float + most_common: list[LengthEntry] + + +class PatternEntry(BaseModel): + """A masked character pattern and how many sampled rows matched it.""" + + model_config = ConfigDict(extra="allow") + + pattern: str + count: int + + +class PatternComposition(BaseModel): + """The proportion of letter, digit, and other characters across sampled values.""" + + model_config = ConfigDict(extra="allow") + + letter: float + digit: float + other: float + + +class PatternsStatistics(BaseModel): + """The most common character patterns and the overall character composition.""" + + model_config = ConfigDict(extra="allow") + + top: list[PatternEntry] + composition: PatternComposition + + +class MaskedFormEntry(BaseModel): + """A masked leading form of a value and how many sampled rows had it.""" + + model_config = ConfigDict(extra="allow") + + masked: str + count: int + + +class FirstCharsStatistics(BaseModel): + """The most common masked leading forms.""" + + model_config = ConfigDict(extra="allow") + + top: list[MaskedFormEntry] + + +class StringStatistics(BaseModel): + """String preview statistics; `patterns`/`first_chars` appear at or above certain disclosure levels.""" + + model_config = ConfigDict(extra="allow") + + lengths: LengthsStatistics + patterns: Optional[PatternsStatistics] = None + first_chars: Optional[FirstCharsStatistics] = None + + +class NumericSummaries(BaseModel): + """Numeric mean and percentile summary.""" + + model_config = ConfigDict(extra="allow") + + mean: float + q1: float + q2: float + q3: float + p5: float + p95: float + + +class NumericBin(BaseModel): + """One half-open interval `[lower_bound, upper_bound)` of a numeric histogram and its row count.""" + + model_config = ConfigDict(extra="allow") + + lower_bound: float + upper_bound: float + count: int + + +class NumericHistograms(BaseModel): + """The bins of a numeric histogram.""" + + model_config = ConfigDict(extra="allow") + + bins: list[NumericBin] + + +class NumericStatistics(BaseModel): + """Numeric preview statistics; `histograms` is present only at the histogram disclosure level.""" + + model_config = ConfigDict(extra="allow") + + summaries: NumericSummaries + histograms: Optional[NumericHistograms] = None + + +class TemporalSummaries(BaseModel): + """Temporal mean and percentile summary, each an ISO-formatted string.""" + + model_config = ConfigDict(extra="allow") + + mean: str + q1: str + q2: str + q3: str + p5: str + p95: str + + +class TemporalBin(BaseModel): + """One interval of a temporal histogram, with ISO-formatted bounds and a human-readable label.""" + + model_config = ConfigDict(extra="allow") + + lower_bound: str + upper_bound: str + label: str + count: int + + +class TemporalHistograms(BaseModel): + """The bins of a temporal histogram.""" + + model_config = ConfigDict(extra="allow") + + bins: list[TemporalBin] + + +class TemporalStatistics(BaseModel): + """Temporal preview statistics; `histograms` is present only at the histogram disclosure level.""" + + model_config = ConfigDict(extra="allow") + + summaries: TemporalSummaries + histograms: Optional[TemporalHistograms] = None + + +class BooleanStatistics(BaseModel): + """Counts of true and false values.""" + + model_config = ConfigDict(extra="allow") + + count_true: int + count_false: int + + +class UnsupportedStatistics(BaseModel): + """Why a column could not be previewed.""" + + model_config = ConfigDict(extra="allow") + + reason: str + + +class ColumnPreview(BaseModel): + """Fields present on every Safe Data Preview.""" + + model_config = ConfigDict(extra="allow") + + # For grouped file discovery, the file whose sample produced this preview; None for database columns. + sampled_from: Optional[str] = None + statistics_common: CommonStatistics + + +class StringPreview(ColumnPreview): + """A preview of a string column.""" + + kind: Literal[ColumnKind.string] = ColumnKind.string + statistics_kind: StringStatistics + + +class NumericPreview(ColumnPreview): + """A preview of a numeric column.""" + + kind: Literal[ColumnKind.numeric] = ColumnKind.numeric + statistics_kind: NumericStatistics + + +class TemporalPreview(ColumnPreview): + """A preview of a date/time column.""" + + kind: Literal[ColumnKind.temporal] = ColumnKind.temporal + statistics_kind: TemporalStatistics + + +class BooleanPreview(ColumnPreview): + """A preview of a boolean column.""" + + kind: Literal[ColumnKind.boolean] = ColumnKind.boolean + statistics_kind: BooleanStatistics + + +class UnsupportedPreview(ColumnPreview): + """A column the preview ran against but declined; `statistics_kind.reason` says why.""" + + kind: Literal[ColumnKind.unsupported] = ColumnKind.unsupported + statistics_kind: UnsupportedStatistics + + +SafeDataPreview = Annotated[ + Union[StringPreview, NumericPreview, TemporalPreview, BooleanPreview, UnsupportedPreview], + Field(discriminator="kind"), +] diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 6121430..3318b64 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -26,6 +26,7 @@ SchemaDiscoveryPage, SchemaDiscoveryRequest, SchemaDiscoveryResult, + StringPreview, ) from datamasque.client.exceptions import ( AsyncRulesetGenerationInProgressError, @@ -770,6 +771,23 @@ def _schema_discovery_row(row_id: int, column_name: str, table_name: str = "user } +def test_schema_discovery_result_parses_safe_data_preview(): + row = _schema_discovery_row(1, "email") + row["data"]["safe_data_preview"] = { + "kind": "string", + "sampled_from": "data/people.csv", + "statistics_common": {"count_row": 1000, "count_null": 0, "count_distinct": 988}, + "statistics_kind": { + "lengths": {"min": 5, "max": 30, "mean": 18.0, "median": 18.0, "most_common": []}, + }, + } + result = SchemaDiscoveryResult.model_validate(row) + preview = result.data.safe_data_preview + assert isinstance(preview, StringPreview) + assert preview.sampled_from == "data/people.csv" + assert preview.statistics_common.count_distinct == 988 + + def test_list_schema_discovery_results_follows_pagination(client): run_id = RunId(42) page1 = { @@ -833,6 +851,7 @@ def test_get_schema_discovery_page_returns_page_with_table_metadata(client): "primary_keys": [{"columns": ["id"]}], "unique_keys": [{"columns": ["email"]}], "foreign_keys": [], + "row_count": 12345, }, }, }, @@ -848,6 +867,7 @@ def test_get_schema_discovery_page_returns_page_with_table_metadata(client): assert isinstance(page, SchemaDiscoveryPage) assert [r.column for r in page.results] == ["email"] assert page.table_metadata["public"]["users"].primary_keys[0].columns == ["id"] + assert page.table_metadata["public"]["users"].row_count == 12345 assert m.last_request.qs == {"limit": ["10"], "offset": ["20"]} diff --git a/tests/test_safe_data_preview.py b/tests/test_safe_data_preview.py new file mode 100644 index 0000000..f346a70 --- /dev/null +++ b/tests/test_safe_data_preview.py @@ -0,0 +1,178 @@ +"""Tests for the typed Safe Data Preview models.""" + +import pytest +from pydantic import BaseModel, ValidationError + +from datamasque.client import FileDiscoveryResult, SchemaDiscoveryColumn +from datamasque.client.models.safe_data_preview import ( + NumericPreview, + UnsupportedPreview, +) + +_STRING_PREVIEW = { + "kind": "string", + "sampled_from": None, + "statistics_common": {"count_row": 1000, "count_null": 12, "count_distinct": 988}, + "statistics_kind": { + "lengths": { + "min": 9, + "max": 34, + "mean": 21.4, + "median": 21.0, + "most_common": [{"length": 20, "count": 140}, {"length": 22, "count": 118}], + }, + "patterns": { + "top": [ + {"pattern": "aaaa@aaaa.aaa", "count": 512}, + {"pattern": "aaaa.aaaa@aaaa.aaa", "count": 300}, + ], + "composition": {"letter": 0.78, "digit": 0.05, "other": 0.17}, + }, + "first_chars": {"top": [{"masked": "j*******", "count": 40}, {"masked": "s*******", "count": 33}]}, + }, +} + +_NUMERIC_PREVIEW = { + "kind": "numeric", + "sampled_from": None, + "statistics_common": {"count_row": 1000, "count_null": 0, "count_distinct": 71}, + "statistics_kind": { + "summaries": {"mean": 41.3, "q1": 29.0, "q2": 41.0, "q3": 54.0, "p5": 19.0, "p95": 68.0}, + "histograms": { + "bins": [ + {"lower_bound": 18.0, "upper_bound": 28.0, "count": 210}, + {"lower_bound": 28.0, "upper_bound": 38.0, "count": 240}, + ] + }, + }, +} + +_TEMPORAL_PREVIEW = { + "kind": "temporal", + "sampled_from": None, + "statistics_common": {"count_row": 500, "count_null": 3, "count_distinct": 480}, + "statistics_kind": { + "summaries": { + "mean": "1987-04-12", + "q1": "1975-01-01", + "q2": "1988-06-30", + "q3": "2001-03-15", + "p5": "1960-01-01", + "p95": "2010-12-31", + }, + "histograms": { + "bins": [ + {"lower_bound": "1970-01-01", "upper_bound": "1980-01-01", "label": "1970s", "count": 88}, + {"lower_bound": "1980-01-01", "upper_bound": "1990-01-01", "label": "1980s", "count": 142}, + ] + }, + }, +} + +_BOOLEAN_PREVIEW = { + "kind": "boolean", + "sampled_from": None, + "statistics_common": {"count_row": 1000, "count_null": 0, "count_distinct": 2}, + "statistics_kind": {"count_true": 640, "count_false": 360}, +} + +_UNSUPPORTED_PREVIEW = { + "kind": "unsupported", + "sampled_from": None, + "statistics_common": {"count_row": 0, "count_null": 0, "count_distinct": None}, + "statistics_kind": {"reason": "Binary data isn't previewed"}, +} + +_KNOWN_PREVIEWS = [ + pytest.param(_STRING_PREVIEW, id="string"), + pytest.param(_NUMERIC_PREVIEW, id="numeric"), + pytest.param(_TEMPORAL_PREVIEW, id="temporal"), + pytest.param(_BOOLEAN_PREVIEW, id="boolean"), + pytest.param(_UNSUPPORTED_PREVIEW, id="unsupported"), +] + + +def _assert_no_unexpected_extras(value: object) -> None: + """Recursively assert no field landed in a model's `model_extra` -- i.e. everything is typed.""" + if isinstance(value, BaseModel): + assert not value.model_extra, f"unexpected extra fields on {type(value).__name__}: {value.model_extra}" + for field_value in value.__dict__.values(): + _assert_no_unexpected_extras(field_value) + elif isinstance(value, list): + for item in value: + _assert_no_unexpected_extras(item) + + +def _build_column_data(**overrides: object) -> dict[str, object]: + data: dict[str, object] = { + "data_type": "text", + "foreign_keys": [], + "discovery_matches": [], + "constraint_columns": [], + "unique_index_names": [], + "referencing_foreign_keys": [], + "constraint": "", + } + data.update(overrides) + return data + + +@pytest.mark.parametrize("payload", _KNOWN_PREVIEWS) +def test_each_preview_has_no_untyped_fields(payload): + column = SchemaDiscoveryColumn.model_validate(_build_column_data(safe_data_preview=payload)) + _assert_no_unexpected_extras(column.safe_data_preview) + + +@pytest.mark.parametrize( + "overrides", + [ + pytest.param({"safe_data_preview": None}, id="explicit-null"), + pytest.param({}, id="absent"), + ], +) +def test_column_without_preview_is_none(overrides): + column = SchemaDiscoveryColumn.model_validate(_build_column_data(**overrides)) + assert column.safe_data_preview is None + + +def test_column_unsupported_preview_is_distinct_from_none(): + column = SchemaDiscoveryColumn.model_validate(_build_column_data(safe_data_preview=_UNSUPPORTED_PREVIEW)) + assert column.safe_data_preview is not None + assert isinstance(column.safe_data_preview, UnsupportedPreview) + + +@pytest.mark.parametrize( + "safe_data_preview", + [ + pytest.param({"kind": "geospatial", "statistics_common": {"count_row": 1, "count_null": 0}}, id="unknown-kind"), + pytest.param({"kind": "numeric", "statistics_common": {"count_row": 5, "count_null": 0}}, id="broken-shape"), + pytest.param({"kind": 123, "statistics_common": {"count_row": "oops"}}, id="malformed"), + pytest.param("not a preview", id="non-dict"), + ], +) +def test_column_rejects_unparseable_preview(safe_data_preview): + with pytest.raises(ValidationError): + SchemaDiscoveryColumn.model_validate(_build_column_data(safe_data_preview=safe_data_preview)) + + +def _file_discovery_payload(locator_extra: dict[str, object]) -> dict[str, object]: + return { + "id": 7, + "connection": {"id": "conn-1", "name": "my files"}, + "file_type": "csv", + "files": [{"path": "data/people.csv", "file_type": "csv"}], + "results": [{"locator": "age", "data_types": ["integer"], "matches": [], **locator_extra}], + } + + +def test_file_locator_preview_parses_with_sampled_from(): + payload = _file_discovery_payload({"safe_data_preview": {**_NUMERIC_PREVIEW, "sampled_from": "data/people.csv"}}) + result = FileDiscoveryResult.model_validate(payload) + preview = result.results[0].safe_data_preview + assert isinstance(preview, NumericPreview) + assert preview.sampled_from == "data/people.csv" + + +def test_file_locator_without_preview_is_none(): + result = FileDiscoveryResult.model_validate(_file_discovery_payload({})) + assert result.results[0].safe_data_preview is None