diff --git a/docs/in_memory_api.md b/docs/in_memory_api.md index 40824adb..f265842e 100644 --- a/docs/in_memory_api.md +++ b/docs/in_memory_api.md @@ -29,7 +29,7 @@ import pandas as pd synth = dpsynth.TabularSynthesizer( domains=domains, - discrete_mechanism=discrete_mechanisms.MSTMechanism(), + discrete_mechanism=discrete_mechanisms.MSTConfig(), ) result = synth.calibrate( epsilon=1.0, diff --git a/dpsynth/api.py b/dpsynth/api.py index d1e7ddca..28834c19 100644 --- a/dpsynth/api.py +++ b/dpsynth/api.py @@ -19,7 +19,7 @@ Example usage:: - mechanism = dpsynth.discrete_mechanisms.AIMMechanism(pgm_iters=500) + mechanism = dpsynth.discrete_mechanisms.AIMConfig(pgm_iters=500) # Option 1: Calibrate to (epsilon, delta)-DP (tight PLD accounting). calibrated = mechanism.calibrate(epsilon=1.0, delta=1e-5) @@ -42,24 +42,58 @@ import dp_accounting -class DPMechanism(abc.ABC): - """Abstract base class for differentially private mechanisms. +class CalibratedMechanism(abc.ABC): + """A privacy-calibrated, runnable differentially private mechanism. - A DPMechanism encapsulates a randomized algorithm that satisfies differential - privacy. Usage follows a three-phase pattern: + Produced by ``MechanismConfig.configure()`` / ``.calibrate()``: its natural + privacy parameter (e.g. Gaussian sigma) is populated and it is ready to run. + It exposes the exact ``DpEvent`` characterizing its privacy cost and is + directly callable on data. - 1. **Construct**: Create the mechanism with algorithm-specific parameters - (e.g., ``AIMMechanism(pgm_iters=500)``). + Subclasses must implement: + + - ``dp_event``: return the exact ``DpEvent`` characterizing the mechanism. + - ``__call__``: run the mechanism on data. + """ + + @property + @abc.abstractmethod + def dp_event(self) -> dp_accounting.DpEvent: + """The DpEvent characterizing the privacy cost of this mechanism.""" + + @abc.abstractmethod + def __call__(self, *args: Any, **kwargs: Any) -> Any: + """Runs the mechanism on the given data. + + Subclass signatures vary, but typically accept at least the data to operate + on and a source of randomness. + + Args: + *args: Positional arguments (subclass-specific). + **kwargs: Keyword arguments (subclass-specific). + """ + + +class MechanismConfig(abc.ABC): + """A recipe that produces a calibrated, runnable mechanism. + + A config holds the mechanism's structural and hyperparameter fields (the + parts a user writes, logs, and serializes) and knows how to turn a privacy + budget into a runnable ``CalibratedMechanism``. Usage follows a three-phase + pattern: + + 1. **Construct**: Create the config with algorithm-specific parameters + (e.g., ``AIMConfig(pgm_iters=500)``). 2. **Calibrate**: Call ``calibrate(epsilon=..., delta=...)`` or - ``configure(zcdp_rho=...)`` to bind a privacy budget, returning a new - frozen instance with the mechanism's natural privacy parameter set. + ``configure(zcdp_rho=...)`` to bind a privacy budget, returning a + ``CalibratedMechanism`` with the mechanism's natural privacy parameter set. 3. **Run**: Call the calibrated mechanism on data via ``__call__``. **Design: configure vs calibrate.** The API separates two concerns: - ``configure(zcdp_rho, **kwargs)`` is the low-level primitive that each - mechanism must implement. It maps a zCDP budget to the mechanism's natural - privacy parameter (e.g., Gaussian sigma) and returns a new frozen instance. + config must implement. It maps a zCDP budget to the mechanism's natural + privacy parameter (e.g., Gaussian sigma) and returns a runnable mechanism. This is lightweight — just arithmetic — and produces reasonably tight parameter settings for most mechanisms. @@ -82,24 +116,17 @@ class DPMechanism(abc.ABC): search exploits this: it evaluates each candidate's raw ``dp_event`` rather than relying on the zCDP conversion, so the final calibration is as tight as the mechanism's own privacy characterization allows. - - Subclasses must implement: - - - ``configure(zcdp_rho, **kwargs)``: set the mechanism's natural privacy - parameter (e.g., Gaussian sigma) from a zCDP budget. - - ``dp_event``: return the exact ``DpEvent`` characterizing the mechanism. - - ``__call__``: run the mechanism on data. """ @abc.abstractmethod def configure( - self, *, zcdp_rho: float, delta: float = 0.0, **kwargs: Any - ) -> DPMechanism: - """Returns a new mechanism configured with the given zCDP budget. + self, *, zcdp_rho, delta=0.0, max_records_per_user=1 + ) -> CalibratedMechanism: + """Returns a calibrated mechanism for the given zCDP budget. Converts the zCDP budget into the mechanism's natural privacy parameter - (e.g., Gaussian sigma) and returns a new frozen instance with that - parameter set. + (e.g., Gaussian sigma) and returns a runnable ``CalibratedMechanism`` with + that parameter set. Most mechanisms are pure zCDP and ignore ``delta``. Mechanisms that consume approximate DP budget (e.g., partition selection with Gaussian @@ -111,27 +138,11 @@ def configure( delta: Approximate DP delta consumed by the mechanism itself (e.g., for thresholding). Defaults to 0 (pure zCDP). Mechanisms that need delta should raise if it is 0. - **kwargs: Mechanism-specific hyperparameters. + **kwargs: Mechanism-specific hyperparameters (e.g. + ``max_records_per_user`` for mechanisms that support user-level DP). Returns: - A new DPMechanism instance configured for the given budget. - """ - - @property - @abc.abstractmethod - def dp_event(self) -> dp_accounting.DpEvent: - """The DpEvent characterizing the privacy cost of this mechanism.""" - - @abc.abstractmethod - def __call__(self, *args: Any, **kwargs: Any) -> Any: - """Runs the mechanism on the given data. - - Subclass signatures vary, but typically accept at least the data to operate - on and a source of randomness. - - Args: - *args: Positional arguments (subclass-specific). - **kwargs: Keyword arguments (subclass-specific). + A calibrated, runnable mechanism. """ def _find_optimal_rho( @@ -193,7 +204,7 @@ def calibrate( delta: float | None = None, zcdp_rho: float | None = None, **kwargs: Any, - ) -> DPMechanism: + ) -> CalibratedMechanism: """Calibrate the mechanism to a target (epsilon, delta)-DP guarantee. Performs a binary search over zCDP budgets, calling ``configure`` at each @@ -211,7 +222,7 @@ def calibrate( **kwargs: Forwarded to ``configure()``. Returns: - A new calibrated DPMechanism instance. + A calibrated, runnable mechanism. Raises: ValueError: If neither (epsilon, delta) nor zcdp_rho is specified, or @@ -243,6 +254,20 @@ def calibrate( return self.configure(zcdp_rho=optimal_rho, delta=delta, **kwargs) +class DPMechanism(MechanismConfig, CalibratedMechanism, abc.ABC): + """Transitional monolithic base: both a config and a runnable mechanism. + + Historically every mechanism was a single class that was constructed, then + ``configure``d in place, then run. That design is being split into a + ``MechanismConfig`` recipe and a ``CalibratedMechanism`` runnable (so a + calibrated instance can never be un-calibrated and needs no nullable + privacy-parameter fields or guards). Mechanisms are migrated one layer at a + time; those not yet split still subclass this monolith, which exposes exactly + the historical abstract surface (``configure`` + ``dp_event`` + ``__call__``, + with ``calibrate`` inherited). Remove once every mechanism is split. + """ + + def validate_max_records_per_user(value: int) -> None: """Raises ValueError if the per-user record bound is not a positive int.""" if value < 1: diff --git a/dpsynth/data_generation_v2.py b/dpsynth/data_generation_v2.py index 5f58eaef..d239988d 100644 --- a/dpsynth/data_generation_v2.py +++ b/dpsynth/data_generation_v2.py @@ -35,8 +35,9 @@ def generate( delta: float, *, discrete_config: ( - discrete_mechanisms.DiscreteMechanism - ) = discrete_mechanisms.MSTMechanism(), + discrete_mechanisms.DiscreteMechanismConfig + | discrete_mechanisms.DiscreteMechanism + ) = discrete_mechanisms.MSTConfig(), numerical_bins: int = 32, one_way_marginal_budget_fraction: float = 0.1, cross_attribute_constraints: Sequence[constraints.Constraint] = (), diff --git a/dpsynth/data_generation_v3.py b/dpsynth/data_generation_v3.py index e199482d..4f6fb8e2 100644 --- a/dpsynth/data_generation_v3.py +++ b/dpsynth/data_generation_v3.py @@ -21,7 +21,6 @@ from absl import logging import dp_accounting -from dpsynth import api from dpsynth import constraints from dpsynth import discrete_mechanisms from dpsynth import domain @@ -38,7 +37,6 @@ def _create_initializers( domains: Mapping[str, domain.AttributeType], numerical_bins: int, init_delta: float, - max_records_per_user: int = 1, ) -> dict[str, primitives.DPMechanism]: """Creates per-column initializers from the domain specification. @@ -46,10 +44,6 @@ def _create_initializers( domains: Mapping from column names to attribute domain specifications. numerical_bins: Number of bins for numerical discretization. init_delta: Delta for open-set categorical partition selection. - max_records_per_user: Assumed upper bound on the number of records a single - user contributes. Sensitivity (and hence added noise) is scaled by this - factor to provide user-level rather than record-level DP; the privacy - analysis is unchanged. Soundness relies on the caller enforcing the bound. Returns: A dictionary mapping column names to uncalibrated initializer instances. @@ -60,22 +54,20 @@ def _create_initializers( initializers = {} for col, attr in domains.items(): if isinstance(attr, domain.NumericalAttribute): - initializers[col] = initialization.NumericalInitializer( + initializers[col] = initialization.NumericalInitializerConfig( name=col, num_partitions=numerical_bins, attribute=attr, - max_records_per_user=max_records_per_user, ) elif isinstance(attr, domain.CategoricalAttribute): - initializers[col] = initialization.CategoricalInitializer( - name=col, attribute=attr, max_records_per_user=max_records_per_user + initializers[col] = initialization.CategoricalInitializerConfig( + name=col, attribute=attr ) elif isinstance(attr, domain.OpenSetCategoricalAttribute): - initializers[col] = initialization.OpenSetCategoricalInitializer( + initializers[col] = initialization.OpenSetCategoricalInitializerConfig( name=col, attribute=attr, delta=init_delta, - max_records_per_user=max_records_per_user, ) else: raise ValueError( @@ -225,9 +217,10 @@ class TabularSynthesizer(primitives.DPMechanism): """ domains: Mapping[str, domain.AttributeType] - discrete_mechanism: discrete_mechanisms.DiscreteMechanism = dataclasses.field( - default_factory=discrete_mechanisms.MSTMechanism - ) + discrete_mechanism: ( + discrete_mechanisms.DiscreteMechanismConfig + | discrete_mechanisms.DiscreteMechanism + ) = dataclasses.field(default_factory=discrete_mechanisms.MSTConfig) numerical_bins: int = 32 init_budget_fraction: float = 0.1 initializers: dict[str, primitives.DPMechanism] | None = None @@ -236,13 +229,15 @@ class TabularSynthesizer(primitives.DPMechanism): experimental_max_records_per_user: int = 1 def __post_init__(self): - api.validate_max_records_per_user(self.experimental_max_records_per_user) + if self.experimental_max_records_per_user <= 0: + raise ValueError('experimental_max_records_per_user must be >= 1') def configure( # pyrefly: ignore[bad-override] self, *, zcdp_rho: float, delta: float = 0.0, + max_records_per_user: int | None = None, ) -> TabularSynthesizer: """Returns a copy configured with the given privacy budget. @@ -266,6 +261,11 @@ def configure( # pyrefly: ignore[bad-override] (``init_budget_fraction``) is allocated to partition selection for open-set columns. Must be positive when open-set categorical attributes are present. + max_records_per_user: Upper bound on the number of records a single user + contributes. Values greater than 1 scale the added noise (and mechanism + sensitivity) to provide user-level rather than record-level DP; the + privacy accounting is unchanged. This bound is NOT enforced -- soundness + relies on the caller guaranteeing it via preprocessing. Returns: A new TabularSynthesizer with calibrated sub-mechanisms. @@ -282,6 +282,9 @@ def configure( # pyrefly: ignore[bad-override] 'delta must be positive when open-set categorical attributes are' ' present. It is used for Gaussian partition selection.' ) + if max_records_per_user is None: + max_records_per_user = self.experimental_max_records_per_user + # Split delta across open-set columns, analogous to splitting zcdp_rho. # Under calibrate(), any delta not consumed here is automatically # available for the zCDP-to-(epsilon, delta) conversion, so this @@ -297,38 +300,33 @@ def configure( # pyrefly: ignore[bad-override] self.domains, self.numerical_bins, per_col_delta, - self.experimental_max_records_per_user, ) - elif self.experimental_max_records_per_user > 1: - # The synthesizer's experimental_max_records_per_user is the single - # source of truth: the total-count and discrete mechanisms already use it, - # so propagate it to caller-supplied initializers too. - propagated = {} - for col, init in inits.items(): - propagated[col] = dataclasses.replace( # pytype: disable=wrong-arg-types - init, max_records_per_user=self.experimental_max_records_per_user - ) - inits = propagated init_rho = self.init_budget_fraction * zcdp_rho # +1 for the DPGaussianCount that always measures the total. per_col_rho = init_rho / (len(inits) + 1) discrete_rho = zcdp_rho - init_rho calibrated_inits = { - col: init.configure(zcdp_rho=per_col_rho) for col, init in inits.items() + col: init.configure( + zcdp_rho=per_col_rho, + max_records_per_user=max_records_per_user, + ) + for col, init in inits.items() } - calibrated_total = primitives.DPGaussianCount( - max_records_per_user=self.experimental_max_records_per_user - ).configure(zcdp_rho=per_col_rho) - calibrated_discrete = dataclasses.replace( - self.discrete_mechanism, - max_records_per_user=self.experimental_max_records_per_user, - ).configure(zcdp_rho=discrete_rho) + calibrated_total = primitives.DPGaussianCountConfig().configure( + zcdp_rho=per_col_rho, + max_records_per_user=max_records_per_user, + ) + calibrated_discrete = self.discrete_mechanism.configure( # pyrefly: ignore[missing-attribute] + zcdp_rho=discrete_rho, + max_records_per_user=max_records_per_user, + ) return dataclasses.replace( self, initializers=calibrated_inits, discrete_mechanism=calibrated_discrete, total_count_mechanism=calibrated_total, + experimental_max_records_per_user=max_records_per_user, ) @property @@ -347,7 +345,7 @@ def dp_event(self) -> dp_accounting.DpEvent: ) events = [init.dp_event for init in self.initializers.values()] events.append(self.total_count_mechanism.dp_event) - events.append(self.discrete_mechanism.dp_event) + events.append(self.discrete_mechanism.dp_event) # pyrefly: ignore[missing-attribute] return dp_accounting.ComposedDpEvent(events) def __call__( @@ -385,7 +383,7 @@ def __call__( total_measurement = mbi.LinearMeasurement( np.array([total]), # pyrefly: ignore[bad-argument-type] (), - stddev=k * self.total_count_mechanism.sigma, # pyrefly: ignore[unsupported-operation] + stddev=k * self.total_count_mechanism.sigma, ) results: dict[str, initialization.ColumnMeasurement] = {} @@ -408,7 +406,7 @@ def __call__( mbi_constraints = tuple( c.to_mbi() for c in self.cross_attribute_constraints ) - mechanism_result = self.discrete_mechanism( + mechanism_result = self.discrete_mechanism( # pyrefly: ignore[not-callable] rng, data=discrete, initial_measurements=initial_measurements, diff --git a/dpsynth/discrete_mechanisms/README.md b/dpsynth/discrete_mechanisms/README.md index 35a8754d..14ff1a47 100644 --- a/dpsynth/discrete_mechanisms/README.md +++ b/dpsynth/discrete_mechanisms/README.md @@ -44,31 +44,31 @@ multiple mechanisms; keep selection policy in individual mechanism files. ## `independent.py` - One-Way Baseline -Implements `IndependentMechanism`, the simplest mechanism. It uses the shared +Implements `Independent`, the simplest mechanism. It uses the shared workflow from `base.py`, spends its budget on one-way marginals, and selects no additional cliques. The resulting model preserves each column's distribution but does not model relationships between columns. -**Public API:** `IndependentMechanism` +**Public API:** `Independent` ## `direct.py` — Caller-Defined Workload -Implements `DirectMechanism`, which measures the cliques supplied through +Implements `Direct`, which measures the cliques supplied through `prespecified_marginal_queries`. It performs no data-dependent selection and does not create its own one-way measurements, so the full budget is available for the specified workload. Initial measurements supplied by another layer are included when fitting the final model. -**Public API:** `DirectMechanism(prespecified_marginal_queries=...)` +**Public API:** `DirectConfig(prespecified_marginal_queries=...)` ## `mst.py` — Private Pairwise Spanning-Tree Selection -Implements `MSTMechanism`, the default general-purpose mechanism for preserving +Implements `MST`, the default general-purpose mechanism for preserving pairwise relationships. It begins with one-way marginals, privately selects pairwise cliques forming a maximum spanning tree, measures those cliques, and uses the shared base pipeline to estimate the final model. -**Public API:** `MSTMechanism` +**Public API:** `MST` **Internal behavior:** `_allocate_budget()` splits remaining rho between private selection and measurement; `_select()` calls the spanning-tree selection logic. @@ -77,11 +77,11 @@ the private pairwise-selection step. ## `aim.py` — Adaptive Iterative Selection -Implements `AIMMechanism`, an adaptive workload-based mechanism. Instead of +Implements `AIM`, an adaptive workload-based mechanism. Instead of selecting cliques once, it repeatedly finds a marginal that the current model approximates poorly, measures it, and updates the model. -**Public API:** `AIMMechanism(workload=...)` +**Public API:** `AIMConfig(workload=...)` **Internal behavior:** `_one_way_cliques()` limits initial measurements to the workload; `_allocate_budget()` reserves rho for the adaptive loop; `_run()` @@ -102,12 +102,12 @@ scores and select the next workload marginal. ## `swift.py` — Workload and Clique-Tree Mechanism -Implements `SWIFTMechanism`, a workload-informed mechanism that selects +Implements `SWIFT`, a workload-informed mechanism that selects marginals while controlling clique-tree complexity. It uses a custom junction-tree-aware estimation and sampling path rather than the standard one-pass implementation in `base.py`. -**Public API:** `SWIFTMechanism(workload=...)` +**Public API:** `SWIFTConfig(workload=...)` **Internal behavior:** `_allocate_budget()` splits rho between selection and measurement; `_run()` compiles the workload, selects supported cliques, builds a diff --git a/dpsynth/discrete_mechanisms/__init__.py b/dpsynth/discrete_mechanisms/__init__.py index 8912fd7b..4b5a29c7 100644 --- a/dpsynth/discrete_mechanisms/__init__.py +++ b/dpsynth/discrete_mechanisms/__init__.py @@ -16,20 +16,27 @@ # pylint: disable=g-importing-member -from dpsynth.discrete_mechanisms.aim import AIMMechanism -from dpsynth.discrete_mechanisms.aim_gdp import AIMGDPMechanism +from dpsynth.discrete_mechanisms.aim import AIM +from dpsynth.discrete_mechanisms.aim import AIMConfig +from dpsynth.discrete_mechanisms.aim_gdp import AIMGDP +from dpsynth.discrete_mechanisms.aim_gdp import AIMGDPConfig from dpsynth.discrete_mechanisms.base import DiscreteMechanism +from dpsynth.discrete_mechanisms.base import DiscreteMechanismConfig from dpsynth.discrete_mechanisms.common import DiscreteMechanismResult from dpsynth.discrete_mechanisms.common import MechanismDiagnostics -from dpsynth.discrete_mechanisms.direct import DirectMechanism -from dpsynth.discrete_mechanisms.independent import IndependentMechanism -from dpsynth.discrete_mechanisms.mst import MSTMechanism -from dpsynth.discrete_mechanisms.swift import SWIFTMechanism +from dpsynth.discrete_mechanisms.direct import Direct +from dpsynth.discrete_mechanisms.direct import DirectConfig +from dpsynth.discrete_mechanisms.independent import Independent +from dpsynth.discrete_mechanisms.independent import IndependentConfig +from dpsynth.discrete_mechanisms.mst import MST +from dpsynth.discrete_mechanisms.mst import MSTConfig +from dpsynth.discrete_mechanisms.swift import SWIFT +from dpsynth.discrete_mechanisms.swift import SWIFTConfig # Backwards-compatible aliases. -AIMConfig = AIMMechanism -AIMGDPConfig = AIMGDPMechanism -DirectConfig = DirectMechanism -IndependentConfig = IndependentMechanism -MSTConfig = MSTMechanism -SWIFTConfig = SWIFTMechanism +AIMMechanism = AIMConfig +AIMGDPMechanism = AIMGDPConfig +DirectMechanism = DirectConfig +IndependentMechanism = IndependentConfig +MSTMechanism = MSTConfig +SWIFTMechanism = SWIFTConfig diff --git a/dpsynth/discrete_mechanisms/aim.py b/dpsynth/discrete_mechanisms/aim.py index 5451cd40..5e53962b 100644 --- a/dpsynth/discrete_mechanisms/aim.py +++ b/dpsynth/discrete_mechanisms/aim.py @@ -86,8 +86,8 @@ def _worst_approximated( return keys[idx] -@dataclasses.dataclass -class AIMMechanism(base.DiscreteMechanism): +@dataclasses.dataclass(frozen=True) +class AIMConfig(base.DiscreteMechanismConfig): """Configuration for the AIM mechanism. Details are described in the paper: @@ -122,7 +122,6 @@ class AIMMechanism(base.DiscreteMechanism): anneal_factor: float = 4.0 select_budget_fraction: float = 0.1 pgm_iters: int = 1000 - _loop_rho: float | None = dataclasses.field(default=None, repr=False) def supporting_cliques(self, domain: mbi.Domain) -> list[mbi.Clique]: """Returns the workload cliques filtered by max_marginal_size.""" @@ -138,10 +137,20 @@ def _allocate_budget(self, remaining_rho: float) -> Mapping[str, float]: """Allocates the entire remaining budget to the adaptive loop.""" return {'_loop_rho': remaining_rho} + def _create_mechanism(self, **kwargs) -> 'AIM': + return AIM(**kwargs) + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class AIM(base.DiscreteMechanism): + """Calibrated AIM instance.""" + + config: AIMConfig + _loop_rho: float + @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the DP event for the AIM mechanism.""" - self._check_calibration() events = self._one_way_dp_event() events.append(dp_accounting.ZCDpEvent(self._loop_rho)) # pyrefly: ignore[bad-argument-type] return dp_accounting.ComposedDpEvent(events) @@ -152,21 +161,24 @@ def _run(self, rng, data, measurements, constraints, phase_times): zcdp_rho = self.zcdp_rho terminate = False rho_remaining = self._loop_rho - max_rounds = self.max_rounds or 16 * len(data.domain) + max_rounds = self.config.max_rounds or 16 * len(data.domain) rho_per_round = self._loop_rho / max_rounds # pyrefly: ignore[unsupported-operation] ######################################################################### # Compile workload into candidate measurements, and precompute answers. # ######################################################################### candidates = common.compiled_workload( - data.domain, self.workload, self.max_marginal_size + data.domain, self.config.workload, self.config.max_marginal_size ) answers = mbi.CliqueVector.from_projectable(data, list(candidates)) # pyrefly: ignore[bad-argument-type] logging.info('[AIM]: Calculated workload-query answers.') - estimator = mbi.estimation.MirrorDescent(self.marginal_oracle) + estimator = mbi.estimation.MirrorDescent(self.config.marginal_oracle) model = estimator.estimate( - data.domain, measurements, iters=self.pgm_iters, constraints=constraints + data.domain, + measurements, + iters=self.config.pgm_iters, + constraints=constraints, ) assert isinstance(model, mbi.MarkovRandomField) @@ -183,10 +195,10 @@ def _run(self, rng, data, measurements, constraints, phase_times): ######################################################################## with common.timed(phase_times, 'selection'): rho_remaining -= rho_per_round # pyrefly: ignore[unsupported-operation] - fraction = self.select_budget_fraction + fraction = self.config.select_budget_fraction sigma = accounting.zcdp_gaussian_sigma((1 - fraction) * rho_per_round) # pyrefly: ignore[unsupported-operation] epsilon = accounting.zcdp_exponential_eps(fraction * rho_per_round) # pyrefly: ignore[unsupported-operation] - size_limit = self.max_model_size * (zcdp_rho - rho_remaining) / zcdp_rho # pyrefly: ignore[unsupported-operation] + size_limit = self.config.max_model_size * (zcdp_rho - rho_remaining) / zcdp_rho # pyrefly: ignore[unsupported-operation] small_candidates = _filter_candidates(candidates, model, size_limit) estimates = mbi.marginal_oracles.bulk_variable_elimination( @@ -243,7 +255,7 @@ def _run(self, rng, data, measurements, constraints, phase_times): data.domain, measurements, potentials=warm_start, - iters=self.pgm_iters, + iters=self.config.pgm_iters, callback_fn=callback_fn, constraints=constraints, ) @@ -262,8 +274,8 @@ def _run(self, rng, data, measurements, constraints, phase_times): ) if np.linalg.norm(new_estimate - old_estimate, ord=1) <= threshold: # No useful information at this noise level, increase budget per round. - rho_per_round *= self.anneal_factor # pyrefly: ignore[unsupported-operation] - fraction = self.select_budget_fraction + rho_per_round *= self.config.anneal_factor # pyrefly: ignore[unsupported-operation] + fraction = self.config.select_budget_fraction sigma = accounting.zcdp_gaussian_sigma((1 - fraction) * rho_per_round) logging.info('[AIM] Reducing sigma: %.1f', sigma) diff --git a/dpsynth/discrete_mechanisms/aim_gdp.py b/dpsynth/discrete_mechanisms/aim_gdp.py index 625525ff..5493e65e 100644 --- a/dpsynth/discrete_mechanisms/aim_gdp.py +++ b/dpsynth/discrete_mechanisms/aim_gdp.py @@ -143,8 +143,8 @@ def _worst_approximated( # select loop, injecting the budgeting strategy (zCDP vs. GDP) as configuration. -@dataclasses.dataclass -class AIMGDPMechanism(base.DiscreteMechanism): +@dataclasses.dataclass(frozen=True) +class AIMGDPConfig(base.DiscreteMechanismConfig): """Configuration for the AIM mechanism with Gaussian DP. Details are described in the paper: @@ -186,7 +186,6 @@ class AIMGDPMechanism(base.DiscreteMechanism): anneal_factor: float = 4.0 select_budget_fraction: float = 0.1 pgm_iters: int = 1000 - _loop_rho: float | None = dataclasses.field(default=None, repr=False) def supporting_cliques(self, domain: mbi.Domain) -> list[mbi.Clique]: """Returns the workload cliques filtered by max_marginal_size.""" @@ -202,10 +201,20 @@ def _allocate_budget(self, remaining_rho: float) -> Mapping[str, float]: """Allocates the entire remaining budget to the adaptive loop.""" return {'_loop_rho': remaining_rho} + def _create_mechanism(self, **kwargs) -> 'AIMGDP': + return AIMGDP(**kwargs) + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class AIMGDP(base.DiscreteMechanism): + """Calibrated AIMGDP instance.""" + + config: AIMGDPConfig + _loop_rho: float + @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the DP event for the AIM-GDP mechanism.""" - self._check_calibration() events = self._one_way_dp_event() # The loop's privacy cost in zCDP terms. events.append(dp_accounting.ZCDpEvent(self._loop_rho)) # pyrefly: ignore[bad-argument-type] @@ -220,22 +229,25 @@ def _run(self, rng, data, measurements, constraints, phase_times): terminate = False budget_remaining = gdp_budget - max_rounds = self.max_rounds or 16 * len(data.domain) + max_rounds = self.config.max_rounds or 16 * len(data.domain) budget_per_round = budget_remaining / max_rounds ######################################################################### # Compile workload into candidate measurements, and precompute answers. # ######################################################################### candidates = common.compiled_workload( - data.domain, self.workload, self.max_marginal_size + data.domain, self.config.workload, self.config.max_marginal_size ) answers = mbi.CliqueVector.from_projectable(data, candidates) # pyrefly: ignore[bad-argument-type] logging.info('[AIM] Calculated workload-query answers.') domain = data.domain - estimator = mbi.estimation.MirrorDescent(self.marginal_oracle) + estimator = mbi.estimation.MirrorDescent(self.config.marginal_oracle) model = estimator.estimate( - domain, measurements, iters=self.pgm_iters, constraints=constraints + domain, + measurements, + iters=self.config.pgm_iters, + constraints=constraints, ) assert isinstance(model, mbi.MarkovRandomField) logging.info('[AIM] Estimated initial model.') @@ -244,12 +256,9 @@ def _run(self, rng, data, measurements, constraints, phase_times): # independence model. compute_independence_errors is much faster than # bulk_variable_elimination for this case (pure numpy, no XLA compilation). budget_remaining -= 0.5 * budget_per_round - per_candidate_sigma = ( + per_candidate_sigma = int( self.max_records_per_user - * accounting.gdp_gaussian_sigma( - 0.5 * budget_per_round / len(candidates) - ) - ) + ) * accounting.gdp_gaussian_sigma(0.5 * budget_per_round / len(candidates)) errors = common.compute_independence_errors(data, model, list(candidates)) # pyrefly: ignore[bad-argument-type] for cl in errors: errors[cl] += rng.normal(loc=0.0, scale=per_candidate_sigma) @@ -268,11 +277,13 @@ def _run(self, rng, data, measurements, constraints, phase_times): ######################################################################## with common.timed(phase_times, 'selection'): budget_remaining -= budget_per_round - measure_budget = budget_per_round * (1 - self.select_budget_fraction) - select_budget = budget_per_round * self.select_budget_fraction + measure_budget = budget_per_round * ( + 1 - self.config.select_budget_fraction + ) + select_budget = budget_per_round * self.config.select_budget_fraction measure_sigma = accounting.gdp_gaussian_sigma(measure_budget) percent_used = (gdp_budget - budget_remaining) / gdp_budget - size_limit = self.max_model_size * percent_used + size_limit = self.config.max_model_size * percent_used small_candidates = _filter_candidates(candidates, model, size_limit) marginal_query = _worst_approximated( @@ -283,7 +294,7 @@ def _run(self, rng, data, measurements, constraints, phase_times): model=model, select_budget=select_budget, measure_sigma=measure_sigma, - max_new_evals=self.max_candidates_per_round, + max_new_evals=self.config.max_candidates_per_round, max_records_per_user=self.max_records_per_user, ) @@ -323,7 +334,7 @@ def _run(self, rng, data, measurements, constraints, phase_times): domain, measurements, potentials=warm_start, - iters=self.pgm_iters, + iters=self.config.pgm_iters, callback_fn=callback_fn, constraints=constraints, ) @@ -344,7 +355,7 @@ def _run(self, rng, data, measurements, constraints, phase_times): ) if np.linalg.norm(new_estimate - old_estimate, ord=1) <= threshold: # No useful information at this noise level, increase budget per round. - budget_per_round *= self.anneal_factor + budget_per_round *= self.config.anneal_factor logging.info( '[AIM] Increasing budget per round: %.5f', budget_per_round ) diff --git a/dpsynth/discrete_mechanisms/base.py b/dpsynth/discrete_mechanisms/base.py index f1764148..559abd88 100644 --- a/dpsynth/discrete_mechanisms/base.py +++ b/dpsynth/discrete_mechanisms/base.py @@ -41,14 +41,16 @@ # hard-codes `mbi.estimation.MirrorDescent` as the estimator; abstract this # (e.g. an injectable optimizer strategy) to support other synthesizers such as # PrivSyn and GEM. -@dataclasses.dataclass -class DiscreteMechanism(api.DPMechanism): + + +@dataclasses.dataclass(frozen=True) +class DiscreteMechanismConfig(api.MechanismConfig): """Base class for mechanisms following the select-measure-estimate paradigm. Subclasses implement ``_select`` to define which marginals to measure. The base ``__call__`` orchestrates the full pipeline:: - check_calibration → measure_one_way → compress → run → result + measure_one_way → compress → run → result where ``_run`` performs select → measure → estimate → generate. One-shot mechanisms need only override ``_select``; adaptive mechanisms (e.g. AIM) or @@ -59,27 +61,12 @@ class DiscreteMechanism(api.DPMechanism): pgm_iters: Number of mirror descent iterations for estimation. compress_columns: Domain compression config. True = all, list = specific. one_way_budget_fraction: Fraction of zCDP budget for one-way marginals. - max_records_per_user: Assumed upper bound on the number of records a single - user contributes. Added noise (and mechanism sensitivity) is scaled by - this factor to provide user-level rather than record-level DP; the privacy - accounting is unchanged. Soundness relies on the caller enforcing this - bound. - zcdp_rho: Total zCDP budget (set by configure). - one_way_rho: zCDP budget for one-way measurements (set by configure). - measurement_rho: zCDP budget for selected marginal measurements. """ marginal_oracle: mbi.MarginalOracle | None = None pgm_iters: int = 5000 compress_columns: bool | Sequence[str] = False one_way_budget_fraction: float = 1 / 3 - max_records_per_user: int = 1 - zcdp_rho: float | None = None - one_way_rho: float | None = dataclasses.field(default=None, repr=False) - measurement_rho: float | None = dataclasses.field(default=None, repr=False) - - def __post_init__(self): - api.validate_max_records_per_user(self.max_records_per_user) @abc.abstractmethod def supporting_cliques(self, domain: mbi.Domain) -> list[mbi.Clique]: @@ -91,53 +78,72 @@ def supporting_cliques(self, domain: mbi.Domain) -> list[mbi.Clique]: the mechanism's coverage without having to run it. Args: - domain: The data domain the mechanism will run on. + domain: Information about the tabular columns and their types. Returns: - The list of cliques supported by this mechanism for ``domain``. + A list of cliques. """ + @abc.abstractmethod + def _create_mechanism(self, **kwargs) -> 'DiscreteMechanism': + """Instantiates the calibrated mechanism object.""" + def configure( self, *, zcdp_rho: float, delta: float = 0.0, initial_measurements: Sequence[mbi.LinearMeasurement] | None = None, + max_records_per_user: int = 1, **kwargs, - ) -> DiscreteMechanism: + ) -> 'DiscreteMechanism': """Configures the mechanism with a zCDP budget.""" + if max_records_per_user <= 0: + raise ValueError('max_records_per_user must be positive') if initial_measurements is not None or self.one_way_budget_fraction <= 0: one_way_rho = None else: one_way_rho = zcdp_rho * self.one_way_budget_fraction remaining_rho = zcdp_rho - (one_way_rho or 0.0) - return dataclasses.replace( - self, + + return self._create_mechanism( + config=self, zcdp_rho=zcdp_rho, one_way_rho=one_way_rho, + max_records_per_user=max_records_per_user, **self._allocate_budget(remaining_rho), ) def _allocate_budget(self, remaining_rho: float) -> Mapping[str, float]: - """Splits the post-one-way budget into mechanism-specific rho fields. + """Splits the post-one-way budget into mechanism-specific rho fields.""" + return {} + + +@dataclasses.dataclass(frozen=True) +class DiscreteMechanism(api.CalibratedMechanism): + """Calibrated, runnable select-measure-estimate mechanism.""" + + config: DiscreteMechanismConfig + zcdp_rho: float + one_way_rho: float | None = dataclasses.field(default=None, repr=False) + measurement_rho: float | None = dataclasses.field(default=None, repr=False) + max_records_per_user: int = 1 - Subclasses override this to distribute ``remaining_rho`` across their own - budget fields (e.g. ``measurement_rho``, ``_select_rho``); the returned - mapping is applied as field overrides in ``configure``. + def supporting_cliques(self, domain: mbi.Domain) -> list[mbi.Clique]: + """Returns the cliques whose marginals this mechanism supports. + + These are the cliques that the graphical model produced by this mechanism + is guaranteed to represent for the given domain, i.e. the marginal queries + the resulting synthetic data can answer. Callers use them to reason about + the mechanism's coverage without having to run it. Args: - remaining_rho: zCDP budget left after the shared one-way measurement. + domain: Information about the tabular columns and their types. Returns: - A mapping from dataclass field name to allocated zCDP budget. + A list of cliques. """ - return {} - - @property - def remaining_rho(self): - """zCDP budget remaining after one-way measurements.""" - one_way_rho = 0.0 if self.one_way_rho is None else self.one_way_rho - return self.zcdp_rho - one_way_rho # pyrefly: ignore[unsupported-operation] + return self.config.supporting_cliques(domain) def _one_way_dp_event(self): """DpEvents for the shared one-way measurement ([] if there is none).""" @@ -149,16 +155,11 @@ def _one_way_dp_event(self): ) ] - def _check_calibration(self): - """Raises ValueError if the mechanism has not been configured.""" - if self.zcdp_rho is None: - raise ValueError('Must call calibrate() before using the mechanism.') - def _one_way_cliques(self, data): """Returns the one-way cliques to measure.""" cliques = [(a,) for a in data.domain] if hasattr(data, 'cliques'): - supported = common.downward_closure(data.cliques) # pytype: disable=attribute-error + supported = common.downward_closure(data.cliques) # pyrefly: ignore[attribute-error] cliques = [cl for cl in cliques if cl in supported] return cliques @@ -184,10 +185,10 @@ def _measure_one_way( def _compress(self, data, measurements, constraints): """Compresses the domain by merging rare values.""" mappings = common.compression_mappings( - measurements, self.compress_columns, constraints + measurements, self.config.compress_columns, constraints ) if mappings and hasattr(data, 'compress'): - data = data.compress(mappings) # pytype: disable=attribute-error + data = data.compress(mappings) # pyrefly: ignore[attribute-error] measurements = [m.compress(mappings, data.domain) for m in measurements] return data, measurements, mappings @@ -204,7 +205,6 @@ def __call__( constraints: Sequence[mbi.Constraint] = (), ) -> common.DiscreteMechanismResult: """Runs the select-measure-estimate pipeline.""" - self._check_calibration() phase_times = {} measurements = self._measure_one_way( rng, data, phase_times, initial_measurements=initial_measurements @@ -217,10 +217,10 @@ def __call__( ) if mappings: synthetic_data = synthetic_data.decompress(mappings) - diagnostics = common.clique_stats(model) # pytype: disable=wrong-arg-types + diagnostics = common.clique_stats(model) # pyrefly: ignore[wrong-arg-types] diagnostics.phase_times = phase_times return common.DiscreteMechanismResult( - model=model, # pytype: disable=wrong-arg-types + model=model, # pyrefly: ignore[wrong-arg-types] synthetic_data=synthetic_data, measurements=measurements, diagnostics=diagnostics, @@ -240,7 +240,7 @@ def _run(self, rng, data, measurements, constraints, phase_times): ) # Kick off async AOT compilation of the estimator while we measure. - estimator = mbi.estimation.MirrorDescent(self.marginal_oracle) + estimator = mbi.estimation.MirrorDescent(self.config.marginal_oracle) futures = None try: futures = estimator.precompile( @@ -269,7 +269,7 @@ def _run(self, rng, data, measurements, constraints, phase_times): model = estimator.estimate( data.domain, measurements, - iters=self.pgm_iters, + iters=self.config.pgm_iters, callback_fn=mbi.callbacks.default(measurements, data.domain), constraints=constraints, ) diff --git a/dpsynth/discrete_mechanisms/direct.py b/dpsynth/discrete_mechanisms/direct.py index c088756c..1076341d 100644 --- a/dpsynth/discrete_mechanisms/direct.py +++ b/dpsynth/discrete_mechanisms/direct.py @@ -23,8 +23,8 @@ import mbi -@dataclasses.dataclass -class DirectMechanism(base.DiscreteMechanism): +@dataclasses.dataclass(frozen=True) +class DirectConfig(base.DiscreteMechanismConfig): """Configuration for the direct mechanism. The direct mechanism measures a prespecified set of marginal queries, @@ -54,13 +54,19 @@ def _allocate_budget(self, remaining_rho: float) -> Mapping[str, float]: """Allocates the full remaining budget to the prespecified queries.""" return {'measurement_rho': remaining_rho} + def _create_mechanism(self, **kwargs) -> 'Direct': + return Direct(**kwargs) + + +@dataclasses.dataclass(frozen=True) +class Direct(base.DiscreteMechanism): + config: DirectConfig @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the DP event for the direct mechanism.""" - self._check_calibration() return dp_accounting.GaussianDpEvent( noise_multiplier=accounting.zcdp_gaussian_sigma(self.measurement_rho) # pyrefly: ignore[bad-argument-type] ) def _select(self, rng, data, measurements, phase_times): - return list(self.prespecified_marginal_queries) + return list(self.config.prespecified_marginal_queries) diff --git a/dpsynth/discrete_mechanisms/independent.py b/dpsynth/discrete_mechanisms/independent.py index 1f7b1b9b..41ce05a7 100644 --- a/dpsynth/discrete_mechanisms/independent.py +++ b/dpsynth/discrete_mechanisms/independent.py @@ -22,8 +22,8 @@ import mbi -@dataclasses.dataclass -class IndependentMechanism(base.DiscreteMechanism): +@dataclasses.dataclass(frozen=True) +class IndependentConfig(base.DiscreteMechanismConfig): """Measures only one-way marginals, allocating the entire budget to them.""" one_way_budget_fraction: float = 1.0 @@ -32,10 +32,17 @@ def supporting_cliques(self, domain: mbi.Domain) -> list[mbi.Clique]: """Returns the one-way marginals this mechanism will measure.""" return [(a,) for a in domain.attributes] + def _create_mechanism(self, **kwargs) -> 'Independent': + return Independent(**kwargs) + + +@dataclasses.dataclass(frozen=True) +class Independent(base.DiscreteMechanism): + config: IndependentConfig + @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the DP event for the independent mechanism.""" - self._check_calibration() return dp_accounting.GaussianDpEvent( noise_multiplier=accounting.zcdp_gaussian_sigma(self.one_way_rho) # pyrefly: ignore[bad-argument-type] ) diff --git a/dpsynth/discrete_mechanisms/mst.py b/dpsynth/discrete_mechanisms/mst.py index 769172fd..8f414bed 100644 --- a/dpsynth/discrete_mechanisms/mst.py +++ b/dpsynth/discrete_mechanisms/mst.py @@ -155,8 +155,8 @@ def _select_two_way_marginal_queries( ) -@dataclasses.dataclass -class MSTMechanism(base.DiscreteMechanism): +@dataclasses.dataclass(frozen=True) +class MSTConfig(base.DiscreteMechanismConfig): """Configuration for the maximum spanning tree mechanism. Details are described in the paper: @@ -172,7 +172,6 @@ class MSTMechanism(base.DiscreteMechanism): select_budget_fraction: float = 1 / 3 maximum_marginal_size: int = 10_000_000 - _select_rho: float | None = dataclasses.field(default=None, repr=False) def supporting_cliques(self, domain: mbi.Domain) -> list[mbi.Clique]: """Returns all pairwise marginals within the size limit.""" @@ -190,11 +189,20 @@ def _allocate_budget(self, remaining_rho: float) -> Mapping[str, float]: 'measurement_rho': remaining_rho - select_rho, } + def _create_mechanism(self, **kwargs) -> 'MST': + return MST(**kwargs) + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class MST(base.DiscreteMechanism): + """Calibrated MST instance.""" + + config: MSTConfig + _select_rho: float = -1.0 + @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the DP event for the MST mechanism.""" - if self.zcdp_rho is None: - raise ValueError('Must call calibrate() before using the mechanism.') # exponential mechanisms and (d-1) Gaussian mechanisms. return dp_accounting.ZCDpEvent(self.zcdp_rho) @@ -205,6 +213,6 @@ def _select(self, rng, data, measurements, phase_times): data, self._select_rho, # pyrefly: ignore[bad-argument-type] measurements, - maximum_marginal_size=self.maximum_marginal_size, + maximum_marginal_size=self.config.maximum_marginal_size, max_records_per_user=self.max_records_per_user, ) diff --git a/dpsynth/discrete_mechanisms/swift.py b/dpsynth/discrete_mechanisms/swift.py index 62332d00..606cd2e7 100644 --- a/dpsynth/discrete_mechanisms/swift.py +++ b/dpsynth/discrete_mechanisms/swift.py @@ -45,8 +45,8 @@ import numpy as np -@dataclasses.dataclass -class SWIFTMechanism(base.DiscreteMechanism): +@dataclasses.dataclass(frozen=True) +class SWIFTConfig(base.DiscreteMechanismConfig): """Configuration for the SWIFT mechanism. Attributes: @@ -72,7 +72,6 @@ class SWIFTMechanism(base.DiscreteMechanism): one_way_budget_fraction: float = 0.1 # Internal state set by configure. - _select_rho: float | None = dataclasses.field(default=None, repr=False) def supporting_cliques(self, domain: mbi.Domain) -> list[mbi.Clique]: """Returns the workload cliques filtered by max_marginal_size.""" @@ -88,10 +87,20 @@ def _allocate_budget(self, remaining_rho: float) -> Mapping[str, float]: 'measurement_rho': remaining_rho - select_rho, } + def _create_mechanism(self, **kwargs) -> 'SWIFT': + return SWIFT(**kwargs) + + +@dataclasses.dataclass(frozen=True, kw_only=True) +class SWIFT(base.DiscreteMechanism): + """Calibrated SWIFT instance.""" + + config: SWIFTConfig + _select_rho: float + @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the DP event for the SWIFT mechanism.""" - self._check_calibration() # SWIFT's budget is split between one-way, selection, and measurement. # All three are Gaussian mechanism applications. return dp_accounting.ZCDpEvent(self.zcdp_rho) # pyrefly: ignore[bad-argument-type] @@ -110,8 +119,8 @@ def _run(self, rng, data, measurements, constraints, phase_times): with common.timed(phase_times, 'compiled_workload'): candidates = common.compiled_workload( data.domain, - self.workload, - self.max_marginal_size, + self.config.workload, + self.config.max_marginal_size, ) logging.info('[SWIFT] %d candidates.', len(candidates)) @@ -120,9 +129,12 @@ def _run(self, rng, data, measurements, constraints, phase_times): domain = data.domain with common.timed(phase_times, 'initial_mirror_descent'): - estimator = mbi.estimation.MirrorDescent(self.marginal_oracle) + estimator = mbi.estimation.MirrorDescent(self.config.marginal_oracle) model = estimator.estimate( - domain, measurements, iters=self.pgm_iters, constraints=constraints + domain, + measurements, + iters=self.config.pgm_iters, + constraints=constraints, ) model = typing.cast(mbi.MarkovRandomField, model) @@ -145,7 +157,11 @@ def _run(self, rng, data, measurements, constraints, phase_times): with common.timed(phase_times, 'select_queries'): selected, jtree = select_queries( - errors, candidates, domain, self.max_clique_size, budget_remaining + errors, + candidates, + domain, + self.config.max_clique_size, + budget_remaining, ) all_cliques = [m.clique for m in measurements] + list(selected) @@ -201,7 +217,7 @@ def _run(self, rng, data, measurements, constraints, phase_times): final_model = estimator.estimate( domain, measurements, - iters=self.pgm_iters, + iters=self.config.pgm_iters, callback_fn=callback_fn, constraints=constraints, ) diff --git a/dpsynth/examples/detailed_example_theory_and_in_memory_api.ipynb b/dpsynth/examples/detailed_example_theory_and_in_memory_api.ipynb index 8464bb1a..21a78e2e 100644 --- a/dpsynth/examples/detailed_example_theory_and_in_memory_api.ipynb +++ b/dpsynth/examples/detailed_example_theory_and_in_memory_api.ipynb @@ -213,9 +213,9 @@ "choose_mechanism = \"mst\" # @param [ \"mst\", \"aim\"]\n", "\n", "if choose_mechanism == \"mst\":\n", - " mechanism = dpsynth.discrete_mechanisms.MSTMechanism()\n", + " mechanism = dpsynth.discrete_mechanisms.MSTConfig()\n", "else:\n", - " mechanism = dpsynth.discrete_mechanisms.AIMMechanism()\n", + " mechanism = dpsynth.discrete_mechanisms.AIMConfig()\n", "\n", "print(\"Started generating synthetic data, this may take a while...\")\n", "import warnings\n", diff --git a/dpsynth/examples/quickstart.ipynb b/dpsynth/examples/quickstart.ipynb index bd89422e..fb4a6802 100644 --- a/dpsynth/examples/quickstart.ipynb +++ b/dpsynth/examples/quickstart.ipynb @@ -112,7 +112,7 @@ "# 3. Configure and calibrate the synthesis mechanism (MST as default)\n", "mechanism = data_generation_v3.TabularSynthesizer(\n", " domains=attribute_domains,\n", - " discrete_mechanism=discrete_mechanisms.MSTMechanism(),\n", + " discrete_mechanism=discrete_mechanisms.MSTConfig(),\n", ").calibrate(epsilon=1.0, delta=1e-5)\n", "\n", "# 4. Generate Differentially Private synthetic data\n", diff --git a/dpsynth/local_mode/beam_adapter.py b/dpsynth/local_mode/beam_adapter.py index d794c7d2..1c3d8c51 100644 --- a/dpsynth/local_mode/beam_adapter.py +++ b/dpsynth/local_mode/beam_adapter.py @@ -74,6 +74,12 @@ Row = dict[str, Any] Initializer = ( + initialization.NumericalInitializerConfig + | initialization.CategoricalInitializerConfig + | initialization.OpenSetCategoricalInitializerConfig +) + +CalibratedInitializer = ( initialization.NumericalInitializer | initialization.CategoricalInitializer | initialization.OpenSetCategoricalInitializer @@ -89,20 +95,20 @@ def __init__(self, initializers: dict[str, Initializer]): super().__init__() self._specs: list[tuple[str, str, dict[str, Any]]] = [] for column, init in initializers.items(): - if isinstance(init, initialization.NumericalInitializer): + if isinstance(init, initialization.NumericalInitializerConfig): attr = init.attribute - lower, upper, gs = init._grid_spec + lower, upper, gs = init.grid_spec delta = (upper - lower) / (gs - 1) meta = dict(attribute=attr, lower=lower, upper=upper, delta=delta) self._specs.append((column, 'numerical', meta)) - elif isinstance(init, initialization.CategoricalInitializer): + elif isinstance(init, initialization.CategoricalInitializerConfig): meta = { 'lookup': init.attribute.lookup, 'default': init.attribute.out_of_domain_index, } self._specs.append((column, 'categorical', meta)) - elif isinstance(init, initialization.OpenSetCategoricalInitializer): + elif isinstance(init, initialization.OpenSetCategoricalInitializerConfig): self._specs.append((column, 'openset', {})) else: raise TypeError(f'Unsupported initializer type: {type(init)}') @@ -161,7 +167,7 @@ def __init__(self, initializers: dict[str, Initializer]): self._openset_min_counts = { col: init.min_count for col, init in initializers.items() - if isinstance(init, initialization.OpenSetCategoricalInitializer) + if isinstance(init, initialization.OpenSetCategoricalInitializerConfig) } def expand( @@ -207,7 +213,7 @@ def _sparse_to_openset(sparse): # mbi) into the Beam pipeline, which can increase setup time for each worker. def run_from_summary( sparse_stats: dict[str, list[tuple[Any, int]]], - initializers: dict[str, Initializer], + initializers: dict[str, CalibratedInitializer], rng: np.random.Generator, ) -> dict[str, initialization.ColumnMeasurement]: """Converts materialized sparse stats to ColumnMeasurements on the driver. @@ -228,10 +234,10 @@ def run_from_summary( for column, init in initializers.items(): sparse = sparse_stats[column] if isinstance(init, initialization.NumericalInitializer): - counts = _sparse_to_dense_numerical(sparse, init.grid_size) + counts = _sparse_to_dense_numerical(sparse, init.config.grid_spec[2]) results[column] = init.from_summary(rng, counts) elif isinstance(init, initialization.CategoricalInitializer): - counts = _sparse_to_dense_categorical(sparse, init.attribute.size) + counts = _sparse_to_dense_categorical(sparse, init.config.attribute.size) results[column] = init.from_summary(rng, counts) elif isinstance(init, initialization.OpenSetCategoricalInitializer): unique_values, value_counts = _sparse_to_openset(sparse) @@ -421,6 +427,7 @@ def generate_from_marginals( initial_measurements = [total_measurement, *codec.one_way_measurements()] mbi_constraints = tuple(c.to_mbi() for c in synth.cross_attribute_constraints) logging.info('[DPSynth/Beam]: Running discrete mechanism.') + # pyrefly: ignore[missing-attribute,not-callable] mechanism_result = synth.discrete_mechanism( rng, data=marginals, @@ -449,7 +456,7 @@ def _run_two_pass( sigma = total_count_mechanism.sigma if total_count_mechanism else None if synth.initializers is None or sigma is None: raise ValueError('TabularSynthesizer must be calibrated.') - inits = cast(dict[str, Initializer], synth.initializers) + inits = cast(dict[str, CalibratedInitializer], synth.initializers) if pipeline_kwargs is None: pipeline_kwargs = {} @@ -466,7 +473,9 @@ def _run_two_pass( rows = create_rows_fn(p) summary = ( rows - | ComputeSufficientStats(inits) + | ComputeSufficientStats( + {name: c.config for name, c in inits.items()} + ) | 'ToDict' >> beam.combiners.ToDict() ) _ = summary | 'WriteSummary' >> beam.Map(_write, path=summary_path) @@ -487,6 +496,7 @@ def _run_two_pass( mbi_domain = data_generation_v3.TabularCodec.from_measurements( column_measurements, synth.domains ).mbi_domain + # pyrefly: ignore[missing-attribute] workload = synth.discrete_mechanism.supporting_cliques(mbi_domain) # Pass 2: compute the marginal workload. diff --git a/dpsynth/local_mode/initialization.py b/dpsynth/local_mode/initialization.py index 9bafc7b7..d24f13c2 100644 --- a/dpsynth/local_mode/initialization.py +++ b/dpsynth/local_mode/initialization.py @@ -70,47 +70,22 @@ class ColumnMeasurement: measurement: mbi.LinearMeasurement | None = None -def _validate_mechanism(mechanism: _M | None) -> _M: - """Validates that the mechanism has been calibrated and returns it.""" - if mechanism is None: - raise ValueError('Must call calibrate() before using the mechanism.') - return mechanism - - -@dataclasses.dataclass -class NumericalInitializer(primitives.DPMechanism): - """Mechanism that creates the data encoding transform for numerical data. - - Internally delegates to a ``DPQuantiles`` mechanism for privacy accounting - and quantile computation. - - Attributes: - name: Attribute name used as the clique key in the measurement. - num_partitions: Number of quantile partitions (must be a power of 2). - attribute: The NumericalAttribute defining the data domain. - max_records_per_user: Assumed upper bound on the number of records a single - user contributes. Added noise (and mechanism sensitivity) is scaled by - this factor to provide user-level rather than record-level DP; the privacy - accounting is unchanged. Soundness relies on the caller enforcing this - bound. - """ +@dataclasses.dataclass(frozen=True) +class NumericalInitializerConfig(api.MechanismConfig): + """Configuration for a numerical data encoding mechanism.""" name: str num_partitions: int attribute: domain.NumericalAttribute max_grid_size: int = 10_000_000 - max_records_per_user: int = 1 - mechanism: primitives.DPQuantiles | None = dataclasses.field( - default=None, repr=False - ) + epsilon_ratio: float = 2.0 def __post_init__(self): - api.validate_max_records_per_user(self.max_records_per_user) if self.max_grid_size < 2: raise ValueError(f'max_grid_size must be >= 2, got {self.max_grid_size}.') @property - def _grid_spec(self) -> tuple[float, float, int]: + def grid_spec(self) -> tuple[float, float, int]: """Returns (lower, upper, grid_size) for the quantile candidate grid.""" attr = self.attribute if attr.dtype == 'int': @@ -126,28 +101,37 @@ def _grid_spec(self) -> tuple[float, float, int]: @property def grid_size(self) -> int: """Grid size used for histogram construction.""" - return self._grid_spec[2] - - def configure( # pyrefly: ignore[bad-override] - self, *, zcdp_rho: float, delta: float = 0.0, epsilon_ratio: float = 2.0 - ) -> NumericalInitializer: - """Returns a copy calibrated to the given zCDP budget.""" - lower, upper, _ = self._grid_spec - mechanism = primitives.DPQuantiles( + return self.grid_spec[2] + + def configure(self, *, zcdp_rho, delta=0.0, max_records_per_user=1): + """Returns a runnable mechanism calibrated to the given zCDP budget.""" + lower, upper, _ = self.grid_spec + mechanism = primitives.DPQuantilesConfig( num_partitions=self.num_partitions, lower=lower, upper=upper, jitter_strategy=( 'refine' if self.attribute.dtype == 'int' else 'symmetric' ), - max_records_per_user=self.max_records_per_user, - ).configure(zcdp_rho=zcdp_rho, epsilon_ratio=epsilon_ratio) - return dataclasses.replace(self, mechanism=mechanism) + epsilon_ratio=self.epsilon_ratio, + ).configure( + zcdp_rho=zcdp_rho, + max_records_per_user=max_records_per_user, + ) + return NumericalInitializer(config=self, mechanism=mechanism) + + +@dataclasses.dataclass(frozen=True) +class NumericalInitializer(api.CalibratedMechanism): + """Mechanism that creates the data encoding transform for numerical data.""" + + config: NumericalInitializerConfig + mechanism: primitives.DPQuantiles @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the composed privacy event for the quantile computation.""" - return _validate_mechanism(self.mechanism).dp_event + return self.mechanism.dp_event def __call__( self, @@ -156,27 +140,16 @@ def __call__( *, estimated_total: float | None = None, ) -> ColumnMeasurement: - """Returns a ColumnMeasurement with the discretization transform. - - Args: - rng: A numpy random number generator. - data: 1D array of numerical data. - estimated_total: If provided, a heuristic one-way measurement is included - assuming a uniform distribution over the original bins. - - Returns: - A ColumnMeasurement with bin edges and optionally a heuristic measurement. - """ - _validate_mechanism(self.mechanism) + """Returns a ColumnMeasurement with the discretization transform.""" counts = self._grid_histogram(data) return self.from_summary(rng, counts, estimated_total=estimated_total) def _grid_histogram(self, data): """Returns the quantile candidate-grid histogram (length grid_size).""" # Applies NumericalAttribute.standardize semantics in a vectorized manner. - lower, upper, gs = self._grid_spec + lower, upper, gs = self.config.grid_spec delta = (upper - lower) / (gs - 1) - attr = self.attribute + attr = self.config.attribute values = np.asarray(data, dtype=float) if attr.clip_to_range: values = np.where(np.isnan(values), attr.min_value, values) @@ -196,15 +169,14 @@ def from_summary( estimated_total: float | None = None, ) -> ColumnMeasurement: """Returns a ColumnMeasurement from pre-aggregated histogram counts.""" - mechanism = _validate_mechanism(self.mechanism) - raw_edges = mechanism(rng, counts) + raw_edges = self.mechanism(rng, counts) return edges_to_column_measurement( raw_edges=raw_edges, - attribute=self.attribute, - name=self.name, - zcdp_rho=mechanism.zcdp_rho, + attribute=self.config.attribute, + name=self.config.name, + zcdp_rho=self.mechanism.zcdp_rho, estimated_total=estimated_total, - max_records_per_user=self.max_records_per_user, + max_records_per_user=self.mechanism.max_records_per_user, ) @@ -269,120 +241,83 @@ def edges_to_column_measurement( return ColumnMeasurement(cat_attr, bin_edges, measurement=measurement) -@dataclasses.dataclass -class CategoricalInitializer(primitives.DPMechanism): - """Mechanism that measures a noisy histogram for categorical data. - - Internally delegates to a ``DPGaussianHistogram`` mechanism for privacy - accounting and noise addition. - - Attributes: - name: Attribute name used as the clique key in the measurement. - attribute: The CategoricalAttribute defining the closed domain. - max_records_per_user: Assumed upper bound on the number of records a single - user contributes. Added noise (and mechanism sensitivity) is scaled by - this factor to provide user-level rather than record-level DP; the privacy - accounting is unchanged. Soundness relies on the caller enforcing this - bound. - """ +@dataclasses.dataclass(frozen=True) +class CategoricalInitializerConfig(api.MechanismConfig): + """Configuration for measuring a noisy histogram for categorical data.""" name: str attribute: domain.CategoricalAttribute - max_records_per_user: int = 1 - mechanism: primitives.DPGaussianHistogram | None = dataclasses.field( - default=None, repr=False - ) - - def __post_init__(self): - api.validate_max_records_per_user(self.max_records_per_user) - def configure( # pyrefly: ignore[bad-override] - self, *, zcdp_rho: float, delta: float = 0.0 - ) -> CategoricalInitializer: - """Returns a copy calibrated to the given zCDP budget.""" - mechanism = primitives.DPGaussianHistogram( + def configure(self, *, zcdp_rho, delta=0.0, max_records_per_user=1): # pyrefly: ignore[bad-override] + """Returns a runnable mechanism calibrated to the given zCDP budget.""" + mechanism = primitives.DPGaussianHistogramConfig( domain_size=self.attribute.size, - max_records_per_user=self.max_records_per_user, - ).configure(zcdp_rho=zcdp_rho) - return dataclasses.replace(self, mechanism=mechanism) + ).configure(zcdp_rho=zcdp_rho, max_records_per_user=max_records_per_user) + return CategoricalInitializer(config=self, mechanism=mechanism) + + +@dataclasses.dataclass(frozen=True) +class CategoricalInitializer(api.CalibratedMechanism): + """Mechanism that measures a noisy histogram for categorical data.""" + + config: CategoricalInitializerConfig + mechanism: primitives.DPGaussianHistogram @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the Gaussian privacy event for this mechanism.""" - return _validate_mechanism(self.mechanism).dp_event + return self.mechanism.dp_event def __call__( self, rng: np.random.Generator, data: np.ndarray ) -> ColumnMeasurement: """Returns a ColumnMeasurement with the noisy histogram.""" - encoded = vtx.discrete_encode(data, self.attribute) - counts = np.bincount(encoded, minlength=self.attribute.size) + encoded = vtx.discrete_encode(data, self.config.attribute) + counts = np.bincount(encoded, minlength=self.config.attribute.size) return self.from_summary(rng, counts) def from_summary( self, rng: np.random.Generator, counts: np.ndarray ) -> ColumnMeasurement: """Returns a ColumnMeasurement from pre-aggregated counts.""" - mechanism = _validate_mechanism(self.mechanism) - result = mechanism(rng, counts) + result = self.mechanism(rng, counts) measurement = mbi.LinearMeasurement( result.counts, - (self.name,), - stddev=mechanism.max_records_per_user * mechanism.sigma, # pyrefly: ignore[unsupported-operation] + (self.config.name,), + stddev=self.mechanism.max_records_per_user * self.mechanism.sigma, ) - return ColumnMeasurement(self.attribute, measurement=measurement) + return ColumnMeasurement(self.config.attribute, measurement=measurement) -@dataclasses.dataclass -class OpenSetCategoricalInitializer(primitives.DPMechanism): - """Mechanism that discovers and measures an open-set categorical domain. - - Uses Gaussian Thresholding (Algorithm 2 from the DP-SIPS paper) to privately - select significant partitions from the data and simultaneously obtain noisy - counts for each discovered partition. The discovered partitions, together - with the attribute's default_value (used as a catch-all for undiscovered - values), form a CategoricalAttribute used for downstream synthesis. - - Attributes: - name: Attribute name used as the clique key in the measurement. - attribute: The OpenSetCategoricalAttribute specifying the default value. - delta: Failure probability for the partition selection threshold. - min_count: Minimum true count for a partition to be discovered. - max_records_per_user: Assumed upper bound on the number of records a single - user contributes. Added noise (and mechanism sensitivity) is scaled by - this factor to provide user-level rather than record-level DP; the privacy - accounting is unchanged. Soundness relies on the caller enforcing this - bound. - """ +@dataclasses.dataclass(frozen=True) +class OpenSetCategoricalInitializerConfig(api.MechanismConfig): + """Configuration for discovering an open-set categorical domain.""" name: str attribute: domain.OpenSetCategoricalAttribute delta: float min_count: int = 1 - max_records_per_user: int = 1 - mechanism: primitives.DPPartitionSelection | None = dataclasses.field( - default=None, repr=False - ) - def __post_init__(self): - api.validate_max_records_per_user(self.max_records_per_user) - - def configure( # pyrefly: ignore[bad-override] - self, *, zcdp_rho: float, delta: float = 0.0 - ) -> OpenSetCategoricalInitializer: - """Returns a copy calibrated to the given zCDP budget.""" - # max_records_per_user > 1 is supported via a naive, conservative threshold. - mechanism = primitives.DPPartitionSelection( + def configure(self, *, zcdp_rho, delta=0.0, max_records_per_user=1): # pyrefly: ignore[bad-override] + """Returns a runnable mechanism calibrated to the given zCDP budget.""" + mechanism = primitives.DPPartitionSelectionConfig( delta=self.delta, min_count=self.min_count, - max_records_per_user=self.max_records_per_user, - ).configure(zcdp_rho=zcdp_rho) - return dataclasses.replace(self, mechanism=mechanism) + ).configure(zcdp_rho=zcdp_rho, max_records_per_user=max_records_per_user) + return OpenSetCategoricalInitializer(config=self, mechanism=mechanism) + + +@dataclasses.dataclass(frozen=True) +class OpenSetCategoricalInitializer(api.CalibratedMechanism): + """Mechanism that discovers and measures an open-set categorical domain.""" + + config: OpenSetCategoricalInitializerConfig + mechanism: primitives.DPPartitionSelection @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the privacy event including thresholding delta.""" - return _validate_mechanism(self.mechanism).dp_event + return self.mechanism.dp_event def __call__( self, rng: np.random.Generator, data: np.ndarray @@ -399,14 +334,13 @@ def from_summary( counts: np.ndarray, ) -> ColumnMeasurement: """Returns a ColumnMeasurement from pre-aggregated value counts.""" - mechanism = _validate_mechanism(self.mechanism) - result = mechanism.from_summary(rng, counts) + result = self.mechanism.from_summary(rng, counts) selected_values = [ str(v) for v in unique_values[result.selected_partitions] ] # Build the discovered domain: default first, then selected values. - possible_values = [self.attribute.default_value] + selected_values + possible_values = [self.config.attribute.default_value] + selected_values cat_attr = domain.CategoricalAttribute( possible_values=possible_values, # pyrefly: ignore[unexpected-keyword] out_of_domain_index=0, # pyrefly: ignore[unexpected-keyword] @@ -416,8 +350,8 @@ def from_summary( # not the unmeasured default at index 0. measurement = mbi.LinearMeasurement( result.estimated_counts, # pyrefly: ignore[bad-argument-type] - (self.name,), - stddev=mechanism.max_records_per_user * mechanism.sigma, # pyrefly: ignore[unsupported-operation] + (self.config.name,), + stddev=self.mechanism.max_records_per_user * self.mechanism.sigma, query=mbi.SlicedQuery(start=1), ) return ColumnMeasurement(cat_attr, measurement=measurement) diff --git a/dpsynth/local_mode/primitives.py b/dpsynth/local_mode/primitives.py index baf7c0d7..d3e77c99 100644 --- a/dpsynth/local_mode/primitives.py +++ b/dpsynth/local_mode/primitives.py @@ -30,6 +30,8 @@ import numpy as np import scipy.stats +CalibratedMechanism = api.CalibratedMechanism +MechanismConfig = api.MechanismConfig DPMechanism = api.DPMechanism @@ -48,11 +50,6 @@ class PartitionSelectionResult: estimated_counts: np.ndarray -_UNCALIBRATED_MSG = ( - '{param} has not been set. Set it directly or call calibrate().' -) - - def _contribution_bound(prng, user_ids, max_part): """Return array idx where all ids appear <=max_part times in user_ids[idx].""" # Sort by ID + noise to shuffle within groups. Then find where @@ -278,18 +275,9 @@ def _select_partitions_sips( return selected_partitions, selected_counts, max_sigma -# --------------------------------------------------------------------------- -# DPMechanism subclasses -# --------------------------------------------------------------------------- - - -@dataclasses.dataclass -class DPQuantiles(DPMechanism): - """Differentially private quantiles via composed exponential mechanisms. - - Computes quantile edges by recursive median bisection on a dense histogram. - The ``__call__`` method takes a 1D histogram of counts and returns the - quantile edge values. +@dataclasses.dataclass(frozen=True) +class DPQuantilesConfig(MechanismConfig): + """Recipe for differentially private quantiles. Attributes: num_partitions: Number of quantile partitions (must be a power of 2). @@ -297,24 +285,14 @@ class DPQuantiles(DPMechanism): upper: Upper bound of the data domain (exclusive). jitter_strategy: Tie-breaking jitter passed to ``quantiles_from_histogram``: ``'refine'`` for integer attributes, ``'symmetric'`` for continuous ones. - max_records_per_user: Assumed upper bound on the number of records a single - user contributes. Added noise (and mechanism sensitivity) is scaled by - this factor to provide user-level rather than record-level DP; the privacy - accounting is unchanged. Soundness relies on the caller enforcing this - bound. + epsilon_ratio: Factor by which epsilon grows at each deeper level. """ num_partitions: int lower: float upper: float jitter_strategy: Literal['symmetric', 'refine'] = 'symmetric' - max_records_per_user: int = 1 - _epsilon_levels: tuple[float, ...] | None = dataclasses.field( - default=None, repr=False - ) - - def __post_init__(self): - api.validate_max_records_per_user(self.max_records_per_user) + epsilon_ratio: float = 2.0 @property def _num_levels(self) -> int: @@ -323,74 +301,94 @@ def _num_levels(self) -> int: raise ValueError(f'{self.num_partitions=} must be a power of 2.') return result - @property - def zcdp_rho(self) -> float: - """Total zCDP rho consumed, derived from the per-level epsilons.""" - if self._epsilon_levels is None: - raise ValueError(_UNCALIBRATED_MSG.format(param='_epsilon_levels')) - return sum(e**2 / 8.0 for e in self._epsilon_levels) - - def configure( # pyrefly: ignore[bad-override] - self, *, zcdp_rho: float, delta: float = 0.0, epsilon_ratio: float = 2.0 - ) -> DPQuantiles: - """Returns a copy calibrated to the given zCDP budget. - - Args: - zcdp_rho: The zCDP privacy budget (rho). - delta: Unused. Accepted for interface compatibility. - epsilon_ratio: Factor by which epsilon grows at each deeper level. - """ - if zcdp_rho <= 0: - raise ValueError(f'zcdp_rho must be positive, got {zcdp_rho}.') + def configure(self, *, zcdp_rho, delta=0.0, max_records_per_user=1): # pyrefly: ignore[bad-override] + """Returns a calibrated mechanism for the given zCDP budget.""" levels = self._num_levels - if levels == 0: - return dataclasses.replace(self, _epsilon_levels=()) - rho_ratio = epsilon_ratio**2 + rho_ratio = self.epsilon_ratio**2 budget_weights = rho_ratio ** np.arange(levels)[::-1] rho_levels = zcdp_rho * budget_weights / budget_weights.sum() eps = np.sqrt(8.0 * rho_levels) - return dataclasses.replace(self, _epsilon_levels=tuple(eps.tolist())) + return DPQuantiles(self, tuple(eps.tolist()), max_records_per_user) + + +@dataclasses.dataclass(frozen=True) +class DPQuantiles(CalibratedMechanism): + """Calibrated DP quantiles via composed exponential mechanisms. + + Computes quantile edges by recursive median bisection on a dense histogram. + The ``__call__`` method takes a 1D histogram of counts and returns the + quantile edge values. + + Attributes: + config: The recipe this mechanism was calibrated from. + epsilon_levels: Per-level exponential-mechanism epsilons. + max_records_per_user: Assumed upper bound on the number of records a single + user contributes. The per-level epsilons are divided by this factor at run + time to provide user-level rather than record-level DP; the privacy + accounting is unchanged. Soundness relies on the caller enforcing this + bound. + """ + + config: DPQuantilesConfig + epsilon_levels: tuple[float, ...] + max_records_per_user: int = 1 + + def __post_init__(self): + api.validate_max_records_per_user(self.max_records_per_user) + + @property + def zcdp_rho(self) -> float: + """Total zCDP rho consumed, derived from the per-level epsilons.""" + return sum(e**2 / 8.0 for e in self.epsilon_levels) @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the composed privacy event for the quantile computation.""" - if self._epsilon_levels is None: - raise ValueError(_UNCALIBRATED_MSG.format(param='_epsilon_levels')) return dp_accounting.ComposedDpEvent([ dp_accounting.ExponentialMechanismDpEvent(epsilon=float(eps)) - for eps in self._epsilon_levels + for eps in self.epsilon_levels ]) def __call__( self, rng: np.random.Generator, counts: np.ndarray ) -> list[float]: """Returns quantile edges from a dense histogram of counts.""" - if self._epsilon_levels is None: - raise ValueError(_UNCALIBRATED_MSG.format(param='_epsilon_levels')) + eps_levels = np.asarray(self.epsilon_levels) / self.max_records_per_user indices = _quantiles.quantiles_from_histogram( - rng, - counts, - epsilon_levels=( - np.asarray(self._epsilon_levels) / self.max_records_per_user - ), - jitter_strategy=self.jitter_strategy, + rng, counts, eps_levels, self.config.jitter_strategy ) # Map cell indices back to domain values; delta is the grid step, which # equals the integer step for integer attributes so edges stay integer. - delta = (self.upper - self.lower) / max(1, np.asarray(counts).size - 1) - return [self.lower + i * delta for i in indices] + delta = (self.config.upper - self.config.lower) / max(1, counts.size - 1) + return [self.config.lower + i * delta for i in indices] -@dataclasses.dataclass -class DPGaussianHistogram(DPMechanism): - """Differentially private histogram via the Gaussian mechanism. +@dataclasses.dataclass(frozen=True) +class DPGaussianHistogramConfig(MechanismConfig): + """Recipe for a differentially private histogram via the Gaussian mechanism. + + Attributes: + domain_size: Number of categories in the histogram domain. + """ + + domain_size: int + + def configure(self, *, zcdp_rho, delta=0.0, max_records_per_user=1): # pyrefly: ignore[bad-override] + """Returns a calibrated mechanism with sigma derived from the budget.""" + sigma = math.sqrt(0.5 / zcdp_rho) + return DPGaussianHistogram(self, sigma, max_records_per_user) + + +@dataclasses.dataclass(frozen=True) +class DPGaussianHistogram(CalibratedMechanism): + """Calibrated DP histogram via the Gaussian mechanism. The natural privacy parameter is ``sigma``, the noise standard deviation. The conversion from zCDP is ``sigma = sqrt(0.5 / zcdp_rho)``. Attributes: - domain_size: Number of categories in the histogram domain. - sigma: Gaussian noise standard deviation. Set directly or via ``configure``. + config: The recipe this mechanism was calibrated from. + sigma: Gaussian noise standard deviation. max_records_per_user: Assumed upper bound on the number of records a single user contributes. Added noise (and mechanism sensitivity) is scaled by this factor to provide user-level rather than record-level DP; the privacy @@ -398,65 +396,62 @@ class DPGaussianHistogram(DPMechanism): bound. """ - domain_size: int - sigma: float | None = None + config: DPGaussianHistogramConfig + sigma: float max_records_per_user: int = 1 def __post_init__(self): api.validate_max_records_per_user(self.max_records_per_user) - def configure( # pyrefly: ignore[bad-override] - self, *, zcdp_rho: float, delta: float = 0.0 - ) -> DPGaussianHistogram: - """Returns a copy with sigma derived from the zCDP budget.""" - return dataclasses.replace(self, sigma=math.sqrt(0.5 / zcdp_rho)) - @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the Gaussian privacy event for this mechanism.""" - if self.sigma is None: - raise ValueError(_UNCALIBRATED_MSG.format(param='sigma')) return dp_accounting.GaussianDpEvent(noise_multiplier=self.sigma) def __call__( self, rng: np.random.Generator, counts: np.ndarray ) -> HistogramResult: """Adds Gaussian noise to the given counts.""" - if self.sigma is None: - raise ValueError(_UNCALIBRATED_MSG.format(param='sigma')) noise = rng.normal( - scale=self.max_records_per_user * self.sigma, size=self.domain_size + scale=self.max_records_per_user * self.sigma, + size=self.config.domain_size, ) return HistogramResult(counts=counts.astype(float) + noise) -@dataclasses.dataclass -class DPGaussianCount(DPMechanism): - """Differentially private count via the Gaussian mechanism.""" +@dataclasses.dataclass(frozen=True) +class DPGaussianCountConfig(MechanismConfig): + """Recipe for a differentially private count via the Gaussian mechanism.""" + + def configure(self, *, zcdp_rho, delta=0.0, max_records_per_user=1): # pyrefly: ignore[bad-override] + """Returns a calibrated mechanism with sigma derived from the budget.""" + sigma = math.sqrt(0.5 / zcdp_rho) + return DPGaussianCount(sigma, max_records_per_user) + + +@dataclasses.dataclass(frozen=True) +class DPGaussianCount(CalibratedMechanism): + """Calibrated DP count via the Gaussian mechanism. + + Attributes: + sigma: Gaussian noise standard deviation. + max_records_per_user: Assumed upper bound on the number of records a single + user contributes; added noise is scaled by this factor for user-level DP. + """ - sigma: float | None = None + sigma: float max_records_per_user: int = 1 def __post_init__(self): api.validate_max_records_per_user(self.max_records_per_user) - def configure( # pyrefly: ignore[bad-override] - self, *, zcdp_rho: float, delta: float = 0.0 - ) -> DPGaussianCount: - """Returns a copy with sigma derived from the zCDP budget.""" - return dataclasses.replace(self, sigma=math.sqrt(0.5 / zcdp_rho)) - @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the Gaussian privacy event for this mechanism.""" - if self.sigma is None: - raise ValueError(_UNCALIBRATED_MSG.format(param='sigma')) return dp_accounting.GaussianDpEvent(noise_multiplier=self.sigma) def noisy_count(self, rng: np.random.Generator, true_count: int) -> float: """Returns ``true_count`` plus calibrated Gaussian noise.""" - if self.sigma is None: - raise ValueError(_UNCALIBRATED_MSG.format(param='sigma')) return float( true_count + rng.normal(scale=self.max_records_per_user * self.sigma) ) @@ -467,9 +462,27 @@ def __call__(self, rng: np.random.Generator, data: np.ndarray) -> float: return self.noisy_count(rng, len(data)) -@dataclasses.dataclass -class DPPartitionSelection(DPMechanism): - """Differentially private partition selection via Gaussian Thresholding. +@dataclasses.dataclass(frozen=True) +class DPPartitionSelectionConfig(MechanismConfig): + """Recipe for differentially private partition selection. + + Attributes: + delta: Failure probability for the thresholding step. + min_count: Minimum true count for a partition to be returned. + """ + + delta: float + min_count: int = 1 + + def configure(self, *, zcdp_rho, delta=0.0, max_records_per_user=1): # pyrefly: ignore[bad-override] + """Returns a calibrated mechanism with sigma derived from the budget.""" + sigma = math.sqrt(0.5 / zcdp_rho) + return DPPartitionSelection(self, sigma, max_records_per_user) + + +@dataclasses.dataclass(frozen=True) +class DPPartitionSelection(CalibratedMechanism): + """Calibrated DP partition selection via Gaussian Thresholding. Because partition selection is an approximate (delta > 0) mechanism, the ``dp_event`` composes the Gaussian event with an ``EpsilonDeltaDpEvent`` @@ -480,9 +493,8 @@ class DPPartitionSelection(DPMechanism): suboptimal, since tighter bounding would require a user->record mapping. Attributes: - delta: Failure probability for the thresholding step. - min_count: Minimum true count for a partition to be returned. - sigma: Gaussian noise standard deviation. Set directly or via ``configure``. + config: The recipe this mechanism was calibrated from. + sigma: Gaussian noise standard deviation. max_records_per_user: Assumed upper bound on the number of records a single user contributes. Added noise (and mechanism sensitivity) is scaled by this factor to provide user-level rather than record-level DP; the privacy @@ -490,47 +502,36 @@ class DPPartitionSelection(DPMechanism): bound. """ - delta: float - min_count: int = 1 - sigma: float | None = None + config: DPPartitionSelectionConfig + sigma: float max_records_per_user: int = 1 def __post_init__(self): api.validate_max_records_per_user(self.max_records_per_user) - def configure( # pyrefly: ignore[bad-override] - self, *, zcdp_rho: float, delta: float = 0.0 - ) -> DPPartitionSelection: - """Returns a copy with sigma derived from the zCDP budget.""" - return dataclasses.replace(self, sigma=math.sqrt(0.5 / zcdp_rho)) - @property def dp_event(self) -> dp_accounting.DpEvent: """Returns the privacy event including thresholding delta.""" - if self.sigma is None: - raise ValueError(_UNCALIBRATED_MSG.format(param='sigma')) main_event = dp_accounting.GaussianDpEvent(noise_multiplier=self.sigma) - failure_event = dp_accounting.dp_event.EpsilonDeltaDpEvent(0, self.delta) + failure_event = dp_accounting.dp_event.EpsilonDeltaDpEvent( + 0, self.config.delta + ) return dp_accounting.ComposedDpEvent([main_event, failure_event]) def __call__( self, rng: np.random.Generator, data: np.ndarray ) -> PartitionSelectionResult: """Runs partition selection on integer-encoded partition IDs.""" - if self.sigma is None: - raise ValueError(_UNCALIBRATED_MSG.format(param='sigma')) gdp_budget = np.inf if self.sigma == 0.0 else 1.0 / (self.sigma**2) parts, counts, _ = select_partitions_gaussian_thresholding( rng, data, gdp_budget, - self.delta, - min_count=self.min_count, + self.config.delta, + min_count=self.config.min_count, max_records_per_user=self.max_records_per_user, ) - return PartitionSelectionResult( - selected_partitions=parts, estimated_counts=counts - ) + return PartitionSelectionResult(parts, counts) def from_summary( self, @@ -547,9 +548,7 @@ def from_summary( A PartitionSelectionResult with indices into `counts` as the selected_partitions and their noisy counts. """ - if self.sigma is None: - raise ValueError(_UNCALIBRATED_MSG.format(param='sigma')) - above_min = counts >= self.min_count + above_min = counts >= self.config.min_count eligible_idx = np.where(above_min)[0] eligible_counts = counts[above_min].astype(float) stddev = self.max_records_per_user * self.sigma @@ -560,10 +559,7 @@ def from_summary( # Tight shift: one user can push a partition's count to # (min_count - 1) + max_records_per_user (see that function for the # full derivation). - base = float(self.max_records_per_user + self.min_count - 1) - threshold = base + stddev * scipy.stats.norm.ppf(1.0 - self.delta) + base = float(self.max_records_per_user + self.config.min_count - 1) + threshold = base + stddev * scipy.stats.norm.ppf(1.0 - self.config.delta) passed = noisy_counts >= threshold - return PartitionSelectionResult( - selected_partitions=eligible_idx[passed], - estimated_counts=noisy_counts[passed], - ) + return PartitionSelectionResult(eligible_idx[passed], noisy_counts[passed]) diff --git a/tests/data_generation_v3_test.py b/tests/data_generation_v3_test.py index 9d9d25bc..e782cf8a 100644 --- a/tests/data_generation_v3_test.py +++ b/tests/data_generation_v3_test.py @@ -231,7 +231,7 @@ def test_configure_propagates_k_to_submechanisms(self): self.assertEqual(calibrated.total_count_mechanism.max_records_per_user, k) self.assertEqual(calibrated.discrete_mechanism.max_records_per_user, k) for init in calibrated.initializers.values(): - self.assertEqual(init.max_records_per_user, k) + self.assertEqual(init.mechanism.max_records_per_user, k) def test_dp_event_invariant_to_k(self): base = TabularSynthesizer(domains=self._categorical_domains()).configure( @@ -271,7 +271,7 @@ def test_custom_initializers_inherit_k(self): domains=domains, initializers=inits, experimental_max_records_per_user=2 ).configure(zcdp_rho=100.0) for init in calibrated.initializers.values(): - self.assertEqual(init.max_records_per_user, 2) + self.assertEqual(init.mechanism.max_records_per_user, 2) @parameterized.named_parameters(('zero', 0), ('negative', -3)) def test_invalid_k_raises(self, k): diff --git a/tests/discrete_mechanisms/aim_test.py b/tests/discrete_mechanisms/aim_test.py index 40056f8c..a6d50dae 100644 --- a/tests/discrete_mechanisms/aim_test.py +++ b/tests/discrete_mechanisms/aim_test.py @@ -61,7 +61,7 @@ class AIMTest(absltest.TestCase): def test_fits_one_way_marginals_with_aim(self): data = mbi.Dataset.synthetic(mbi.Domain(["a", "b", "c"], [3, 4, 5]), N=1000) workload = [("a",), ("b",), ("c",)] - config = aim.AIMMechanism(workload=workload, max_rounds=4, pgm_iters=500) + config = aim.AIMConfig(workload=workload, max_rounds=4, pgm_iters=500) calibrated = config.configure(zcdp_rho=10000) result = calibrated(np.random.default_rng(0), data) @@ -77,7 +77,7 @@ def test_fits_one_way_marginals_with_aim_gdp(self): data = mbi.Dataset.synthetic(mbi.Domain(["a", "b", "c"], [3, 4, 5]), N=1000) workload = [("a",), ("b",), ("c",)] - config = aim_gdp.AIMGDPMechanism( + config = aim_gdp.AIMGDPConfig( workload=workload, max_rounds=4, pgm_iters=500 ) calibrated = config.configure(zcdp_rho=10000) @@ -90,26 +90,10 @@ def test_fits_one_way_marginals_with_aim_gdp(self): actual = result.model.project([col]).datavector() np.testing.assert_allclose(actual, expected, atol=1) - def test_uncalibrated_aim_raises(self): - config = aim.AIMMechanism() - with self.assertRaisesRegex(ValueError, "calibrate"): - _ = config.dp_event - data = mbi.Dataset.synthetic(mbi.Domain(["a"], [3]), N=10) - with self.assertRaisesRegex(ValueError, "calibrate"): - config(np.random.default_rng(0), data) - - def test_uncalibrated_aim_gdp_raises(self): - config = aim_gdp.AIMGDPMechanism() - with self.assertRaisesRegex(ValueError, "calibrate"): - _ = config.dp_event - data = mbi.Dataset.synthetic(mbi.Domain(["a"], [3]), N=10) - with self.assertRaisesRegex(ValueError, "calibrate"): - config(np.random.default_rng(0), data) - def test_correlated_workload_regression_with_aim(self): workload = [("a",), ("b",), ("c",), ("a", "b"), ("a", "c"), ("b", "c")] - config = aim.AIMMechanism(workload=workload, max_rounds=4, pgm_iters=500) - baseline_config = independent.IndependentMechanism(pgm_iters=500) + config = aim.AIMConfig(workload=workload, max_rounds=4, pgm_iters=500) + baseline_config = independent.IndependentConfig(pgm_iters=500) mechanism_error, baseline_error = ( _correlated_workload_mechanism_baseline_errors( config, baseline_config, workload @@ -119,10 +103,10 @@ def test_correlated_workload_regression_with_aim(self): def test_correlated_workload_regression_with_aim_gdp(self): workload = [("a",), ("b",), ("c",), ("a", "b"), ("a", "c"), ("b", "c")] - config = aim_gdp.AIMGDPMechanism( + config = aim_gdp.AIMGDPConfig( workload=workload, max_rounds=4, pgm_iters=500 ) - baseline_config = independent.IndependentMechanism(pgm_iters=500) + baseline_config = independent.IndependentConfig(pgm_iters=500) mechanism_error, baseline_error = ( _correlated_workload_mechanism_baseline_errors( config, baseline_config, workload @@ -131,10 +115,10 @@ def test_correlated_workload_regression_with_aim_gdp(self): self.assertLess(mechanism_error, 0.05 * baseline_error) def test_default_configuration_values(self): - config = aim.AIMMechanism() + config = aim.AIMConfig() self.assertEqual(config.pgm_iters, 1000) - gdp_config = aim_gdp.AIMGDPMechanism() + gdp_config = aim_gdp.AIMGDPConfig() self.assertEqual(gdp_config.pgm_iters, 1000) diff --git a/tests/discrete_mechanisms/base_test.py b/tests/discrete_mechanisms/base_test.py index f3f3315a..cf098ba7 100644 --- a/tests/discrete_mechanisms/base_test.py +++ b/tests/discrete_mechanisms/base_test.py @@ -12,13 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for the shared ``DiscreteMechanism`` base-class machinery. - -These tests exercise the base class in isolation via minimal concrete -subclasses, rather than relying on inherited coverage from child integration -tests. This lets us cover edge cases in the shared boilerplate (budget -splitting, non-fatal precompile failures, and the ``_select`` contract). -""" +"""Unit tests for the shared ``DiscreteMechanism`` base-class machinery.""" import dataclasses from unittest import mock @@ -36,13 +30,10 @@ def _dataset(n: int = 200) -> mbi.Dataset: return mbi.Dataset.synthetic(mbi.Domain(['a', 'b', 'c'], [3, 4, 5]), N=n) -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True, kw_only=True) class _NoOpMechanism(base.DiscreteMechanism): - """Minimal concrete mechanism that selects no additional marginals.""" - @property def dp_event(self) -> dp_accounting.DpEvent: - self._check_calibration() return dp_accounting.GaussianDpEvent(noise_multiplier=1.0) def supporting_cliques(self, domain: mbi.Domain) -> list[mbi.Clique]: @@ -52,68 +43,65 @@ def _select(self, rng, data, measurements, phase_times): return [] -@dataclasses.dataclass -class _NoSelectMechanism(base.DiscreteMechanism): - """Concrete mechanism that (incorrectly) does not override ``_select``.""" +@dataclasses.dataclass(frozen=True, kw_only=True) +class _NoOpMechanismConfig(base.DiscreteMechanismConfig): + + def _create_mechanism(self, **kwargs): + return _NoOpMechanism(**kwargs) + + def supporting_cliques(self, domain): + return [] + +@dataclasses.dataclass(frozen=True, kw_only=True) +class _NoSelectMechanism(base.DiscreteMechanism): @property def dp_event(self) -> dp_accounting.DpEvent: - self._check_calibration() return dp_accounting.GaussianDpEvent(noise_multiplier=1.0) def supporting_cliques(self, domain: mbi.Domain) -> list[mbi.Clique]: return [(a,) for a in domain.attributes] +@dataclasses.dataclass(frozen=True, kw_only=True) +class _NoSelectMechanismConfig(base.DiscreteMechanismConfig): + + def _create_mechanism(self, **kwargs): + return _NoSelectMechanism(**kwargs) + + def supporting_cliques(self, domain): + return [] + + class ConfigureTest(absltest.TestCase): - """Tests for the shared ``configure`` / ``_allocate_budget`` budgeting.""" def test_default_fraction_splits_one_way_budget(self): - configured = _NoOpMechanism(one_way_budget_fraction=0.25).configure( + configured = _NoOpMechanismConfig(one_way_budget_fraction=0.25).configure( zcdp_rho=100.0 ) self.assertEqual(configured.one_way_rho, 25.0) - self.assertEqual(configured.remaining_rho, 75.0) def test_zero_one_way_budget_fraction_skips_one_way(self): - configured = _NoOpMechanism(one_way_budget_fraction=0.0).configure( + configured = _NoOpMechanismConfig(one_way_budget_fraction=0.0).configure( zcdp_rho=100.0 ) self.assertIsNone(configured.one_way_rho) - self.assertEqual(configured.remaining_rho, 100.0) def test_default_allocate_budget_leaves_measurement_rho_unset(self): - # The base ``_allocate_budget`` hook returns an empty mapping, so no - # mechanism-specific budget fields are populated. - configured = _NoOpMechanism().configure(zcdp_rho=100.0) + configured = _NoOpMechanismConfig().configure(zcdp_rho=100.0) self.assertIsNone(configured.measurement_rho) def test_initial_measurements_skip_one_way(self): - configured = _NoOpMechanism(one_way_budget_fraction=0.5).configure( + configured = _NoOpMechanismConfig(one_way_budget_fraction=0.5).configure( zcdp_rho=100.0, initial_measurements=[mock.sentinel.measurement] ) self.assertIsNone(configured.one_way_rho) - self.assertEqual(configured.remaining_rho, 100.0) - - -class CalibrationGuardTest(absltest.TestCase): - """Tests that using an unconfigured mechanism fails fast.""" - - def test_call_without_configure_raises(self): - mechanism = _NoOpMechanism() - with self.assertRaisesRegex(ValueError, 'calibrate'): - mechanism(np.random.default_rng(0), _dataset()) - - def test_dp_event_without_configure_raises(self): - with self.assertRaisesRegex(ValueError, 'calibrate'): - _ = _NoOpMechanism().dp_event class RunMachineryTest(absltest.TestCase): - """Tests for the shared ``_run`` pipeline.""" def test_precompile_failure_is_non_fatal(self): - mechanism = _NoOpMechanism(pgm_iters=100).configure(zcdp_rho=1000.0) + mechanism = _NoOpMechanismConfig(pgm_iters=100).configure(zcdp_rho=1000.0) with mock.patch.object( mbi.estimation.MirrorDescent, 'precompile', @@ -122,13 +110,10 @@ def test_precompile_failure_is_non_fatal(self): result = mechanism(np.random.default_rng(0), _dataset()) mocked_precompile.assert_called_once() self.assertIsInstance(result, common.DiscreteMechanismResult) - self.assertIsNotNone(result.model) def test_missing_select_raises_not_implemented(self): - mechanism = _NoSelectMechanism(pgm_iters=100).configure(zcdp_rho=1000.0) + mechanism = _NoSelectMechanismConfig(pgm_iters=100).configure( + zcdp_rho=1000.0 + ) with self.assertRaises(NotImplementedError): mechanism(np.random.default_rng(0), _dataset()) - - -if __name__ == '__main__': - absltest.main() diff --git a/tests/discrete_mechanisms/direct_test.py b/tests/discrete_mechanisms/direct_test.py index daf06002..0ecf3ed4 100644 --- a/tests/discrete_mechanisms/direct_test.py +++ b/tests/discrete_mechanisms/direct_test.py @@ -25,7 +25,7 @@ def test_fits_one_way_marginals(self): data = mbi.Dataset.synthetic(mbi.Domain(['a', 'b', 'c'], [3, 4, 5]), N=1000) prespecified_queries = [('a', 'b'), ('a', 'c'), ('b', 'c')] - config = direct.DirectMechanism( + config = direct.DirectConfig( prespecified_marginal_queries=prespecified_queries, pgm_iters=500, ) diff --git a/tests/discrete_mechanisms/discrete_mechanisms_test.py b/tests/discrete_mechanisms/discrete_mechanisms_test.py index f7169216..36444bd8 100644 --- a/tests/discrete_mechanisms/discrete_mechanisms_test.py +++ b/tests/discrete_mechanisms/discrete_mechanisms_test.py @@ -32,14 +32,14 @@ _WORKLOAD = [('a', 'b'), ('b', 'c'), ('a',), ('b',), ('c',)] _MECHANISMS = { - 'AIM': aim.AIMMechanism(workload=_WORKLOAD, max_rounds=4, pgm_iters=500), - 'AIM_GDP': aim_gdp.AIMGDPMechanism( + 'AIM': aim.AIMConfig(workload=_WORKLOAD, max_rounds=4, pgm_iters=500), + 'AIM_GDP': aim_gdp.AIMGDPConfig( workload=_WORKLOAD, max_rounds=4, pgm_iters=500 ), - 'MST': mst.MSTMechanism(pgm_iters=500), - 'SWIFT': swift.SWIFTMechanism(workload=_WORKLOAD, pgm_iters=500), - 'Independent': independent.IndependentMechanism(pgm_iters=500), - 'Direct': direct.DirectMechanism( + 'MST': mst.MSTConfig(pgm_iters=500), + 'SWIFT': swift.SWIFTConfig(workload=_WORKLOAD, pgm_iters=500), + 'Independent': independent.IndependentConfig(pgm_iters=500), + 'Direct': direct.DirectConfig( prespecified_marginal_queries=_WORKLOAD, pgm_iters=500 ), } @@ -71,7 +71,7 @@ def test_mechanism_runs_on_precomputed_marginals(self, mechanism): rng = np.random.default_rng(42) calibrated = mechanism.calibrate(zcdp_rho=_ZCDP_RHO) - cliques = calibrated.supporting_cliques(domain) + cliques = mechanism.supporting_cliques(domain) precomputed = mbi.CliqueVector.from_projectable(data, cliques) @@ -145,9 +145,7 @@ def test_dp_event_invariant_to_max_records_per_user(self, mechanism): # Scaling max_records_per_user must not change the accounting: only the # actual noise magnitude scales, while the reported dp_event is identical. base = mechanism.configure(zcdp_rho=_ZCDP_RHO) - scaled = dataclasses.replace(mechanism, max_records_per_user=4).configure( - zcdp_rho=_ZCDP_RHO - ) + scaled = mechanism.configure(zcdp_rho=_ZCDP_RHO, max_records_per_user=4) self.assertEqual(repr(scaled.dp_event), repr(base.dp_event)) @parameterized.named_parameters( @@ -162,9 +160,9 @@ def test_measurement_stddev_scales_with_k(self, mechanism): base = mechanism.configure(zcdp_rho=_ZCDP_RHO)( np.random.default_rng(1), data ) - scaled = dataclasses.replace(mechanism, max_records_per_user=k).configure( - zcdp_rho=_ZCDP_RHO - )(np.random.default_rng(1), data) + scaled = mechanism.configure(zcdp_rho=_ZCDP_RHO, max_records_per_user=k)( + np.random.default_rng(1), data + ) self.assertNotEmpty(base.measurements) self.assertLen(scaled.measurements, len(base.measurements)) for base_m, scaled_m in zip(base.measurements, scaled.measurements): @@ -173,7 +171,7 @@ def test_measurement_stddev_scales_with_k(self, mechanism): @parameterized.named_parameters(('zero', 0), ('negative', -3)) def test_invalid_k_raises(self, k): with self.assertRaises(ValueError): - mst.MSTMechanism(max_records_per_user=k) + mst.MSTConfig().configure(zcdp_rho=_ZCDP_RHO, max_records_per_user=k) if __name__ == '__main__': diff --git a/tests/discrete_mechanisms/independent_test.py b/tests/discrete_mechanisms/independent_test.py index 150e9c47..369cca2b 100644 --- a/tests/discrete_mechanisms/independent_test.py +++ b/tests/discrete_mechanisms/independent_test.py @@ -24,7 +24,7 @@ class IndependentTest(absltest.TestCase): def test_fits_one_way_marginals(self): data = mbi.Dataset.synthetic(mbi.Domain(['a', 'b', 'c'], [3, 4, 5]), N=1000) - config = independent.IndependentMechanism(pgm_iters=500) + config = independent.IndependentConfig(pgm_iters=500) result = config.configure(zcdp_rho=10000)(np.random.default_rng(0), data) self.assertIsInstance(result, common.DiscreteMechanismResult) @@ -35,13 +35,13 @@ def test_fits_one_way_marginals(self): np.testing.assert_allclose(actual, expected, atol=0.1) def test_skips_duplicate_cliques_from_initial_measurements(self): - """IndependentMechanism should not re-measure pre-measured cliques.""" + """Independent should not re-measure pre-measured cliques.""" data = mbi.Dataset.synthetic(mbi.Domain(['a', 'b', 'c'], [3, 4, 5]), N=100) # Pre-measure column 'a'. marginal_a = data.project(('a',)).datavector() initial = [mbi.LinearMeasurement(marginal_a, ('a',), stddev=1.0)] - config = independent.IndependentMechanism(pgm_iters=500) + config = independent.IndependentConfig(pgm_iters=500) # This should not raise 'Cliques must be unique'. model = config.configure(zcdp_rho=100.0)( np.random.default_rng(0), data, initial_measurements=initial diff --git a/tests/discrete_mechanisms/mst_test.py b/tests/discrete_mechanisms/mst_test.py index d381c32d..2788030c 100644 --- a/tests/discrete_mechanisms/mst_test.py +++ b/tests/discrete_mechanisms/mst_test.py @@ -77,7 +77,7 @@ def test_dp_maximum_spanning_tree_infinite_eps(self): def test_fits_one_way_marginals(self): data = mbi.Dataset.synthetic(mbi.Domain(['a', 'b', 'c'], [3, 4, 5]), N=1000) - config = mst.MSTMechanism(pgm_iters=500).configure(zcdp_rho=10000) + config = mst.MSTConfig(pgm_iters=500).configure(zcdp_rho=10000) result = config(np.random.default_rng(0), data) @@ -89,7 +89,7 @@ def test_fits_one_way_marginals(self): np.testing.assert_allclose(actual, expected, atol=1) def test_dp_event_returns_zcdp(self): - config = mst.MSTMechanism().configure(zcdp_rho=1.0) + config = mst.MSTConfig().configure(zcdp_rho=1.0) event = config.dp_event self.assertIsInstance(event, dp_accounting.ZCDpEvent) diff --git a/tests/discrete_mechanisms/swift_test.py b/tests/discrete_mechanisms/swift_test.py index cfe5fc86..365c8fb4 100644 --- a/tests/discrete_mechanisms/swift_test.py +++ b/tests/discrete_mechanisms/swift_test.py @@ -15,7 +15,6 @@ import itertools from absl.testing import absltest -import dp_accounting from dpsynth.discrete_mechanisms import clique_tree from dpsynth.discrete_mechanisms import common from dpsynth.discrete_mechanisms import swift @@ -136,7 +135,7 @@ def test_build_clique_tree(self): def test_fits_one_way_marginals(self): data = mbi.Dataset.synthetic(mbi.Domain(['a', 'b', 'c'], [3, 4, 5]), N=1000) - config = swift.SWIFTMechanism(pgm_iters=500).configure(zcdp_rho=10000) + config = swift.SWIFTConfig(pgm_iters=500).configure(zcdp_rho=10000) result = config(np.random.default_rng(0), data) @@ -147,28 +146,5 @@ def test_fits_one_way_marginals(self): actual = result.model.project([col]).datavector() np.testing.assert_allclose(actual, expected, atol=1) - def test_calibrate_required(self): - config = swift.SWIFTMechanism() - data = mbi.Dataset.synthetic(mbi.Domain(['a', 'b'], [3, 4]), N=100) - with self.assertRaises(ValueError): - config(np.random.default_rng(0), data) - - def test_dp_event_requires_calibration(self): - config = swift.SWIFTMechanism() - with self.assertRaises(ValueError): - _ = config.dp_event - - def test_dp_event_returns_zcdp(self): - config = swift.SWIFTMechanism().configure(zcdp_rho=1.0) - event = config.dp_event - self.assertIsInstance(event, dp_accounting.ZCDpEvent) - - def test_default_configuration_values(self): - config = swift.SWIFTMechanism() - self.assertEqual(config.max_clique_size, 1e7) - self.assertEqual(config.max_marginal_size, 1e6) - self.assertEqual(config.pgm_iters, 10_000) - - if __name__ == '__main__': absltest.main() diff --git a/tests/local_mode/beam_adapter_test.py b/tests/local_mode/beam_adapter_test.py index 5bcf9a14..653d7fc4 100644 --- a/tests/local_mode/beam_adapter_test.py +++ b/tests/local_mode/beam_adapter_test.py @@ -47,12 +47,12 @@ def _rows_fn(rows): class NumericalHistogramTest(absltest.TestCase): def _run(self, rows, attr, max_grid_size=101, num_partitions=4): - init = initialization.NumericalInitializer( + init = initialization.NumericalInitializerConfig( name='x', num_partitions=num_partitions, attribute=attr, max_grid_size=max_grid_size, - ).configure(zcdp_rho=np.inf) + ) _TEST_RESULTS.clear() with beam.Pipeline() as p: stats = ( @@ -65,13 +65,15 @@ def _run(self, rows, attr, max_grid_size=101, num_partitions=4): def _ref_counts(self, values, attr, max_grid_size=101, num_partitions=4): """In-memory grid histogram as an {index: count} dict.""" - init = initialization.NumericalInitializer( + init = initialization.NumericalInitializerConfig( name='x', num_partitions=num_partitions, attribute=attr, max_grid_size=max_grid_size, - ).configure(zcdp_rho=np.inf) - dense = init._grid_histogram(np.asarray(values, dtype=float)) + ) + dense = init.configure(zcdp_rho=np.inf)._grid_histogram( + np.asarray(values, dtype=float) + ) return {i: int(c) for i, c in enumerate(dense) if c} def test_basic_histogram(self): @@ -149,10 +151,10 @@ def test_basic_counts(self): possible_values=['unk', 'a', 'b', 'c'], out_of_domain_index=0, ) - init = initialization.CategoricalInitializer( + init = initialization.CategoricalInitializerConfig( name='col', attribute=attr, - ).configure(zcdp_rho=np.inf) + ) rows = [ {'col': 'a'}, {'col': 'a'}, @@ -182,9 +184,9 @@ class OpenSetCountsTest(absltest.TestCase): def test_basic_counts(self): attr = domain.OpenSetCategoricalAttribute(default_value='') - init = initialization.OpenSetCategoricalInitializer( + init = initialization.OpenSetCategoricalInitializerConfig( name='col', attribute=attr, delta=0.01, min_count=1 - ).configure(zcdp_rho=np.inf) + ) rows = [ {'col': 'apple'}, {'col': 'apple'}, @@ -216,20 +218,14 @@ def test_end_to_end_mixed(self): open_attr = domain.OpenSetCategoricalAttribute(default_value='') initializers = { - 'score': ( - initialization.NumericalInitializer( - name='score', num_partitions=4, attribute=num_attr - ).configure(zcdp_rho=np.inf) + 'score': initialization.NumericalInitializerConfig( + name='score', num_partitions=4, attribute=num_attr ), - 'grade': ( - initialization.CategoricalInitializer( - name='grade', attribute=cat_attr - ).configure(zcdp_rho=np.inf) + 'grade': initialization.CategoricalInitializerConfig( + name='grade', attribute=cat_attr ), - 'tag': ( - initialization.OpenSetCategoricalInitializer( - name='tag', attribute=open_attr, delta=0.01, min_count=1 - ).configure(zcdp_rho=np.inf) + 'tag': initialization.OpenSetCategoricalInitializerConfig( + name='tag', attribute=open_attr, delta=0.01, min_count=1 ), } @@ -250,7 +246,9 @@ def test_end_to_end_mixed(self): ) _ = stats | beam.combiners.ToDict() | beam.Map(_store) measurements = beam_adapter.run_from_summary( - _TEST_RESULTS[0], initializers, rng + _TEST_RESULTS[0], + {k: v.configure(zcdp_rho=np.inf) for k, v in initializers.items()}, + rng, ) self.assertLen(measurements, 3) @@ -263,16 +261,16 @@ class ComputeMarginalsTest(absltest.TestCase): def test_marginals_match_manual_counts(self): cat_attr = domain.CategoricalAttribute(possible_values=['a', 'b', 'c']) num_attr = domain.NumericalAttribute(min_value=0, max_value=10) - cat_init = initialization.CategoricalInitializer( + cat_init = initialization.CategoricalInitializerConfig( name='color', attribute=cat_attr, - ).configure(zcdp_rho=np.inf) - num_init = initialization.NumericalInitializer( + ) + num_init = initialization.NumericalInitializerConfig( name='size', num_partitions=4, attribute=num_attr, max_grid_size=11, - ).configure(zcdp_rho=np.inf) + ) domains = {'color': cat_attr, 'size': num_attr} rows = [ {'color': 'a', 'size': 0}, @@ -294,7 +292,11 @@ def test_marginals_match_manual_counts(self): | beam_adapter.ComputeSufficientStats(inits) ) _ = stats | 'ToDict1' >> beam.combiners.ToDict() | beam.Map(_store) - cms = beam_adapter.run_from_summary(_TEST_RESULTS[0], inits, rng) + cms = beam_adapter.run_from_summary( + _TEST_RESULTS[0], + {k: v.configure(zcdp_rho=np.inf) for k, v in inits.items()}, + rng, + ) # Stage 2: compute marginals. workload = [('color',), ('size',), ('color', 'size')] @@ -378,14 +380,14 @@ def test_end_to_end_mixed_types(self): self.assertCountEqual(result.synthetic_data.columns, ['age', 'grade']) @parameterized.named_parameters( - ('mst', discrete_mechanisms.MSTMechanism(pgm_iters=250)), + ('mst', discrete_mechanisms.MSTConfig(pgm_iters=250)), ( 'independent', - discrete_mechanisms.IndependentMechanism(pgm_iters=250), + discrete_mechanisms.IndependentConfig(pgm_iters=250), ), ( 'direct', - discrete_mechanisms.DirectMechanism( + discrete_mechanisms.DirectConfig( prespecified_marginal_queries=[('a',), ('b',), ('a', 'b')], pgm_iters=250, ), diff --git a/tests/local_mode/initialization_test.py b/tests/local_mode/initialization_test.py index 64e3d786..8f96a5a8 100644 --- a/tests/local_mode/initialization_test.py +++ b/tests/local_mode/initialization_test.py @@ -26,7 +26,7 @@ class InitializationTest(absltest.TestCase): def test_numerical_initializer_dp_event(self): attr = domain.NumericalAttribute(min_value=0, max_value=10) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='test', num_partitions=4, attribute=attr ) event = initializer.configure(zcdp_rho=1.0).dp_event @@ -38,7 +38,7 @@ def test_numerical_initializer_dp_event(self): def test_numerical_initializer_call(self): attr = domain.NumericalAttribute(min_value=0, max_value=10) rng = np.random.default_rng(0) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='test', num_partitions=4, attribute=attr ) @@ -62,7 +62,7 @@ def test_numerical_initializer_deduplicates_bin_edges(self): """Concentrated data can make quantiles return duplicate edges.""" attr = domain.NumericalAttribute(min_value=0, max_value=100) rng = np.random.default_rng(42) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='test', num_partitions=8, attribute=attr ) # Data is heavily concentrated at 50. @@ -83,7 +83,7 @@ def test_numerical_initializer_integer_data(self): """Integer data within a narrow range can collapse quantile edges.""" attr = domain.NumericalAttribute(min_value=0, max_value=10, dtype='int') rng = np.random.default_rng(0) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='test', num_partitions=8, attribute=attr ) # Only 3 distinct values but 8 partitions requested. @@ -102,7 +102,7 @@ def test_numerical_initializer_integer_edges_are_floored(self): """Integer attributes should produce integer-valued bin edges.""" attr = domain.NumericalAttribute(min_value=0, max_value=100, dtype='int') rng = np.random.default_rng(42) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='test', num_partitions=4, attribute=attr ) data = np.arange(100) @@ -117,7 +117,7 @@ def test_numerical_initializer_measurement_with_merged_bins(self): """When integer edges collapse, merged bins get proportionally more mass.""" attr = domain.NumericalAttribute(min_value=0, max_value=100, dtype='int') rng = np.random.default_rng(0) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='test', num_partitions=8, attribute=attr ) # Concentrated data will cause edge collisions. @@ -135,27 +135,27 @@ def test_max_grid_size_below_two_raises(self): attr = domain.NumericalAttribute(min_value=0, max_value=10) for bad in (0, 1): with self.assertRaises(ValueError): - initialization.NumericalInitializer( + initialization.NumericalInitializerConfig( name='x', num_partitions=1, attribute=attr, max_grid_size=bad ) def test_max_grid_size_two_int(self): rng = np.random.default_rng(42) attr = domain.NumericalAttribute(min_value=0, max_value=100, dtype='int') - init = initialization.NumericalInitializer( + init = initialization.NumericalInitializerConfig( name='x', num_partitions=1, attribute=attr, max_grid_size=2 ).configure(zcdp_rho=1.0) - self.assertEqual(init.grid_size, 2) + self.assertEqual(init.config.grid_spec[2], 2) result = init(rng, np.arange(100)) self.assertIsNotNone(result.categorical_attribute) def test_max_grid_size_two_float(self): rng = np.random.default_rng(42) attr = domain.NumericalAttribute(min_value=0.0, max_value=100.0) - init = initialization.NumericalInitializer( + init = initialization.NumericalInitializerConfig( name='x', num_partitions=1, attribute=attr, max_grid_size=2 ).configure(zcdp_rho=1.0) - self.assertEqual(init.grid_size, 2) + self.assertEqual(init.config.grid_spec[2], 2) result = init(rng, np.linspace(0, 100, 100)) self.assertIsNotNone(result.categorical_attribute) @@ -166,16 +166,16 @@ def test_int_grid_reserves_budget_for_jitter_refinement(self): min_value=0, max_value=10_000_000, dtype='int' ) max_grid_size = 100_000 - init = initialization.NumericalInitializer( + init = initialization.NumericalInitializerConfig( name='x', num_partitions=64, attribute=attr, max_grid_size=max_grid_size ) m = _quantiles.jitter_factor(init.num_partitions) - self.assertLessEqual(init.grid_size * m, max_grid_size) + self.assertLessEqual(init.grid_spec[2] * m, max_grid_size) def test_numerical_initializer_measurement_with_estimated_total(self): attr = domain.NumericalAttribute(min_value=0, max_value=10) rng = np.random.default_rng(0) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='num_col', num_partitions=4, attribute=attr ) data = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -198,7 +198,7 @@ def test_numerical_initializer_measurement_with_estimated_total(self): def test_numerical_initializer_no_measurement_without_estimated_total(self): attr = domain.NumericalAttribute(min_value=0, max_value=10) rng = np.random.default_rng(0) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='test', num_partitions=4, attribute=attr ) data = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -209,7 +209,7 @@ def test_integer_edges_at_max_value_absorbed_into_last_bin(self): """Edges at max_value are removed; their count goes to the last bin.""" attr = domain.NumericalAttribute(min_value=0, max_value=10, dtype='int') rng = np.random.default_rng(0) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='test', num_partitions=8, attribute=attr ) # A spread of lower values carrying most of the mass, plus a moderate spike @@ -236,7 +236,7 @@ def test_bin_weights_sum_to_num_partitions(self): attr = domain.NumericalAttribute(min_value=0, max_value=20, dtype='int') for seed in range(10): rng = np.random.default_rng(seed) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='test', num_partitions=8, attribute=attr ) data = np.array([5] * 50 + [15] * 50) @@ -255,7 +255,7 @@ def test_integer_jitter_prevents_spurious_splits(self): """Positive jitter should prevent edges from splitting across integers.""" attr = domain.NumericalAttribute(min_value=0, max_value=100, dtype='int') rng = np.random.default_rng(42) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='test', num_partitions=4, attribute=attr ) # Uniform data: with high budget, edges should land at 25, 50, 75. @@ -270,7 +270,7 @@ def test_integer_heterogeneous_data_buckets(self): """Heterogeneous integer data produces sensible bucket partitioning.""" attr = domain.NumericalAttribute(min_value=0, max_value=10, dtype='int') rng = np.random.default_rng(42) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='x', num_partitions=4, attribute=attr ) # Deliberately lumpy distribution: 45 points across 4 distinct values. @@ -398,7 +398,7 @@ def test_measurement_approximates_true_histogram( self, attr, data, num_partitions, rho ): rng = np.random.default_rng(0) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='x', num_partitions=num_partitions, attribute=attr ) result = initializer.configure(zcdp_rho=rho)( @@ -468,7 +468,7 @@ def test_measurement_property_random_configs(self): # -- Run initializer -- rng = np.random.default_rng(trial) - initializer = initialization.NumericalInitializer( + initializer = initialization.NumericalInitializerConfig( name='x', num_partitions=num_partitions, attribute=attr ) result = initializer.configure(zcdp_rho=rho)( @@ -514,7 +514,7 @@ class CategoricalInitializerTest(absltest.TestCase): def test_dp_event(self): attr = domain.CategoricalAttribute(possible_values=['A', 'B', 'C']) - initializer = initialization.CategoricalInitializer( + initializer = initialization.CategoricalInitializerConfig( name='test', attribute=attr ) event = initializer.configure(zcdp_rho=0.5).dp_event @@ -525,7 +525,7 @@ def test_dp_event(self): def test_call_noiseless(self): attr = domain.CategoricalAttribute(possible_values=['A', 'B', 'C']) rng = np.random.default_rng(0) - initializer = initialization.CategoricalInitializer( + initializer = initialization.CategoricalInitializerConfig( name='col', attribute=attr ) data = np.array(['A', 'A', 'B', 'C', 'C', 'C']) @@ -545,7 +545,7 @@ def test_out_of_domain_values(self): possible_values=['', 'X', 'Y'], out_of_domain_index=0 ) rng = np.random.default_rng(0) - initializer = initialization.CategoricalInitializer( + initializer = initialization.CategoricalInitializerConfig( name='col', attribute=attr ) data = np.array(['X', 'Y', 'Z', 'W']) @@ -561,7 +561,7 @@ class OpenSetCategoricalInitializerTest(absltest.TestCase): def test_dp_event(self): attr = domain.OpenSetCategoricalAttribute(default_value='') - initializer = initialization.OpenSetCategoricalInitializer( + initializer = initialization.OpenSetCategoricalInitializerConfig( name='test', attribute=attr, delta=1e-5 ) event = initializer.configure(zcdp_rho=0.5).dp_event @@ -576,7 +576,7 @@ def test_dp_event(self): def test_call_noiseless(self): attr = domain.OpenSetCategoricalAttribute(default_value='') rng = np.random.default_rng(42) - initializer = initialization.OpenSetCategoricalInitializer( + initializer = initialization.OpenSetCategoricalInitializerConfig( name='col', attribute=attr, delta=1e-5 ) # 'A' appears 100 times, 'B' 50, 'C' 1 (rare). @@ -597,7 +597,7 @@ def test_call_noiseless(self): def test_undiscovered_values_map_to_default(self): attr = domain.OpenSetCategoricalAttribute(default_value='OTHER') rng = np.random.default_rng(0) - initializer = initialization.OpenSetCategoricalInitializer( + initializer = initialization.OpenSetCategoricalInitializerConfig( name='col', attribute=attr, delta=1e-5 ) data = np.array(['A'] * 100 + ['B'] * 50) @@ -615,7 +615,7 @@ def test_undiscovered_values_map_to_default(self): def test_empty_data(self): attr = domain.OpenSetCategoricalAttribute(default_value='') rng = np.random.default_rng(0) - initializer = initialization.OpenSetCategoricalInitializer( + initializer = initialization.OpenSetCategoricalInitializerConfig( name='col', attribute=attr, delta=1e-5 ) data = np.array([], dtype=str) @@ -631,7 +631,7 @@ class NumericalInitializerFromSummaryTest(absltest.TestCase): def test_calibrate_sets_dp_event(self): attr = domain.NumericalAttribute(min_value=0, max_value=100) - init = initialization.NumericalInitializer( + init = initialization.NumericalInitializerConfig( name='age', num_partitions=4, max_grid_size=10001, @@ -642,26 +642,16 @@ def test_calibrate_sets_dp_event(self): # 4 partitions = 2 levels. self.assertLen(event.events, 2) - def test_uncalibrated_raises(self): - attr = domain.NumericalAttribute(min_value=0, max_value=100) - init = initialization.NumericalInitializer( - name='age', - num_partitions=4, - attribute=attr, - ) - with self.assertRaises(ValueError): - init.from_summary(np.random.default_rng(0), np.zeros(100)) - def test_integer_attribute_snaps_edges(self): rng = np.random.default_rng(42) attr = domain.NumericalAttribute(min_value=0, max_value=10, dtype='int') - init = initialization.NumericalInitializer( + init = initialization.NumericalInitializerConfig( name='count', num_partitions=4, attribute=attr, ).configure(zcdp_rho=1.0) # Integer grid: grid_size = 11 (one bin per integer 0..10). - counts = rng.integers(0, 30, size=init.grid_size) + counts = rng.integers(0, 30, size=init.config.grid_spec[2]) cm = init.from_summary(rng, counts) for edge in cm.bin_edges: self.assertEqual(edge, int(edge)) @@ -671,7 +661,7 @@ def test_call_and_from_summary_produce_same_structure(self): rng = np.random.default_rng(42) attr = domain.NumericalAttribute(min_value=0.0, max_value=100.0) max_grid_size = 10001 - init = initialization.NumericalInitializer( + init = initialization.NumericalInitializerConfig( name='x', num_partitions=4, attribute=attr, @@ -696,12 +686,12 @@ class MaxRecordsPerUserTest(parameterized.TestCase): def test_categorical_stddev_scales_with_k(self): attr = domain.CategoricalAttribute(possible_values=['a', 'b', 'c']) data = np.array(['a', 'b', 'c', 'a']) - base = initialization.CategoricalInitializer( + base = initialization.CategoricalInitializerConfig( name='x', attribute=attr ).configure(zcdp_rho=1.0) - scaled = initialization.CategoricalInitializer( - name='x', attribute=attr, max_records_per_user=4 - ).configure(zcdp_rho=1.0) + scaled = initialization.CategoricalInitializerConfig( + name='x', attribute=attr + ).configure(zcdp_rho=1.0, max_records_per_user=4) b = base(np.random.default_rng(0), data) s = scaled(np.random.default_rng(0), data) self.assertAlmostEqual(s.measurement.stddev, 4 * b.measurement.stddev) @@ -709,12 +699,12 @@ def test_categorical_stddev_scales_with_k(self): def test_numerical_heuristic_stddev_scales_with_k(self): attr = domain.NumericalAttribute(min_value=0, max_value=10) data = np.arange(10, dtype=float) - base = initialization.NumericalInitializer( + base = initialization.NumericalInitializerConfig( name='x', num_partitions=4, attribute=attr ).configure(zcdp_rho=1.0) - scaled = initialization.NumericalInitializer( - name='x', num_partitions=4, attribute=attr, max_records_per_user=4 - ).configure(zcdp_rho=1.0) + scaled = initialization.NumericalInitializerConfig( + name='x', num_partitions=4, attribute=attr + ).configure(zcdp_rho=1.0, max_records_per_user=4) b = base(np.random.default_rng(0), data, estimated_total=100.0) s = scaled(np.random.default_rng(0), data, estimated_total=100.0) self.assertAlmostEqual(s.measurement.stddev, 4 * b.measurement.stddev) @@ -722,12 +712,12 @@ def test_numerical_heuristic_stddev_scales_with_k(self): def test_open_set_stddev_scales_with_k(self): attr = domain.OpenSetCategoricalAttribute() data = np.array(['a'] * 50 + ['b'] * 40 + ['c'] * 30) - base = initialization.OpenSetCategoricalInitializer( + base = initialization.OpenSetCategoricalInitializerConfig( name='x', attribute=attr, delta=1e-5 ).configure(zcdp_rho=1.0) - scaled = initialization.OpenSetCategoricalInitializer( - name='x', attribute=attr, delta=1e-5, max_records_per_user=4 - ).configure(zcdp_rho=1.0) + scaled = initialization.OpenSetCategoricalInitializerConfig( + name='x', attribute=attr, delta=1e-5 + ).configure(zcdp_rho=1.0, max_records_per_user=4) b = base(np.random.default_rng(0), data) s = scaled(np.random.default_rng(0), data) self.assertAlmostEqual(s.measurement.stddev, 4 * b.measurement.stddev) @@ -735,10 +725,11 @@ def test_open_set_stddev_scales_with_k(self): @parameterized.named_parameters(('zero', 0), ('negative', -3)) def test_invalid_k_raises(self, k): attr = domain.CategoricalAttribute(possible_values=['a', 'b']) + config = initialization.CategoricalInitializerConfig( + name='x', attribute=attr + ) with self.assertRaises(ValueError): - initialization.CategoricalInitializer( - name='x', attribute=attr, max_records_per_user=k - ) + config.configure(zcdp_rho=0.1, max_records_per_user=k) if __name__ == '__main__': diff --git a/tests/local_mode/primitives_test.py b/tests/local_mode/primitives_test.py index a8207b41..2d0e618d 100644 --- a/tests/local_mode/primitives_test.py +++ b/tests/local_mode/primitives_test.py @@ -127,6 +127,18 @@ def test_mismatched_user_ids_raises(self): ) +def _partition_selection( + delta: float, sigma: float, min_count: int = 1 +) -> primitives.DPPartitionSelection: + """Returns a partition-selection mechanism with an explicit sigma.""" + return primitives.DPPartitionSelection( + config=primitives.DPPartitionSelectionConfig( + delta=delta, min_count=min_count + ), + sigma=sigma, + ) + + class SelectPartitionsGaussianThresholdingTest(absltest.TestCase): def setUp(self): @@ -135,9 +147,7 @@ def setUp(self): def test_basic_operation(self): data = np.array([1] * 50 + [2] * 5) - mech = primitives.DPPartitionSelection( - delta=1e-5, sigma=1.0 / np.sqrt(10.0) - ) + mech = _partition_selection(delta=1e-5, sigma=1.0 / np.sqrt(10.0)) result = mech(self.rng, data) self.assertIn(1, result.selected_partitions) self.assertEqual( @@ -146,14 +156,14 @@ def test_basic_operation(self): def test_empty_data(self): data = np.array([], dtype=int) - mech = primitives.DPPartitionSelection(delta=1e-5, sigma=1.0) + mech = _partition_selection(delta=1e-5, sigma=1.0) result = mech(self.rng, data) self.assertEmpty(result.selected_partitions) self.assertEmpty(result.estimated_counts) def test_high_budget_selects_all(self): data = np.array([1, 2, 3, 4, 5]) - mech = primitives.DPPartitionSelection(delta=0.1, sigma=0.0) + mech = _partition_selection(delta=0.1, sigma=0.0) result = mech(self.rng, data) self.assertCountEqual(result.selected_partitions, [1, 2, 3, 4, 5]) @@ -161,16 +171,14 @@ def test_rare_items_not_selected(self): # One item with many occurrences, another with just 1. # With moderate budget and tight delta, the rare item should be dropped. data = np.array([1] * 100 + [2]) - mech = primitives.DPPartitionSelection(delta=1e-6, sigma=1.0 / np.sqrt(0.5)) + mech = _partition_selection(delta=1e-6, sigma=1.0 / np.sqrt(0.5)) result = mech(self.rng, data) self.assertIn(1, result.selected_partitions) self.assertNotIn(2, result.selected_partitions) def test_string_data_type(self): data = np.array(["a", "b", "a", "a", "c", "a", "c"]) - mech = primitives.DPPartitionSelection( - delta=1e-5, sigma=1.0 / np.sqrt(10.0) - ) + mech = _partition_selection(delta=1e-5, sigma=1.0 / np.sqrt(10.0)) result = mech(self.rng, data) self.assertTrue(all(isinstance(p, str) for p in result.selected_partitions)) @@ -227,9 +235,15 @@ def setUp(self): super().setUp() self.rng = np.random.default_rng(42) + def _histogram(self, domain_size: int, sigma: float): + return primitives.DPGaussianHistogram( + config=primitives.DPGaussianHistogramConfig(domain_size=domain_size), + sigma=sigma, + ) + def test_basic_operation(self): counts = np.array([2, 3, 1, 0]) - mech = primitives.DPGaussianHistogram(domain_size=4, sigma=1.0) + mech = self._histogram(domain_size=4, sigma=1.0) result = mech(self.rng, counts) self.assertLen(result.counts, 4) # Noisy counts should be close to true counts [2, 3, 1, 0]. @@ -237,19 +251,19 @@ def test_basic_operation(self): def test_zero_sigma(self): counts = np.array([2, 1, 3]) - mech = primitives.DPGaussianHistogram(domain_size=3, sigma=0.0) + mech = self._histogram(domain_size=3, sigma=0.0) result = mech(self.rng, counts) np.testing.assert_array_equal(result.counts, [2, 1, 3]) def test_empty_data(self): counts = np.array([0, 0, 0]) - mech = primitives.DPGaussianHistogram(domain_size=3, sigma=1.0) + mech = self._histogram(domain_size=3, sigma=1.0) result = mech(self.rng, counts) self.assertLen(result.counts, 3) # --------------------------------------------------------------------------- -# DPMechanism wrapper tests +# Config + calibrated-mechanism tests class DPGaussianHistogramTest(absltest.TestCase): @@ -259,30 +273,25 @@ def setUp(self): self.rng = np.random.default_rng(42) def test_calibrate_and_call(self): - mech = primitives.DPGaussianHistogram(domain_size=4) - calibrated = mech.configure(zcdp_rho=0.5) + calibrated = primitives.DPGaussianHistogramConfig(domain_size=4).configure( + zcdp_rho=0.5 + ) counts = np.array([2, 3, 1, 0]) result = calibrated(self.rng, counts) self.assertLen(result.counts, 4) np.testing.assert_allclose(result.counts, [2, 3, 1, 0], atol=5.0) def test_direct_sigma(self): - mech = primitives.DPGaussianHistogram(domain_size=3, sigma=0.0) + mech = primitives.DPGaussianHistogram( + config=primitives.DPGaussianHistogramConfig(domain_size=3), sigma=0.0 + ) counts = np.array([2, 1, 3]) np.testing.assert_array_equal(mech(self.rng, counts).counts, [2, 1, 3]) - def test_dp_event_raises_before_calibration(self): - mech = primitives.DPGaussianHistogram(domain_size=4) - with self.assertRaises(ValueError): - _ = mech.dp_event - - def test_call_raises_before_calibration(self): - mech = primitives.DPGaussianHistogram(domain_size=4) - with self.assertRaises(ValueError): - mech(self.rng, np.array([0, 0, 1, 0])) - def test_dp_event_type(self): - mech = primitives.DPGaussianHistogram(domain_size=4).configure(zcdp_rho=0.5) + mech = primitives.DPGaussianHistogramConfig(domain_size=4).configure( + zcdp_rho=0.5 + ) event = mech.dp_event self.assertIsInstance(event, dp_accounting.GaussianDpEvent) self.assertAlmostEqual(event.noise_multiplier, 1.0) @@ -295,8 +304,7 @@ def setUp(self): self.rng = np.random.default_rng(42) def test_calibrate_and_call(self): - mech = primitives.DPGaussianCount() - calibrated = mech.configure(zcdp_rho=0.5) + calibrated = primitives.DPGaussianCountConfig().configure(zcdp_rho=0.5) data = np.array([1, 2, 3, 4, 5]) result = calibrated(self.rng, data) self.assertIsInstance(result, float) @@ -307,13 +315,8 @@ def test_zero_sigma_returns_exact_count(self): data = np.array([10, 20, 30]) self.assertEqual(mech(self.rng, data), 3.0) - def test_dp_event_raises_before_calibration(self): - mech = primitives.DPGaussianCount() - with self.assertRaises(ValueError): - _ = mech.dp_event - def test_dp_event_type(self): - mech = primitives.DPGaussianCount().configure(zcdp_rho=0.5) + mech = primitives.DPGaussianCountConfig().configure(zcdp_rho=0.5) event = mech.dp_event self.assertIsInstance(event, dp_accounting.GaussianDpEvent) self.assertAlmostEqual(event.noise_multiplier, 1.0) @@ -325,10 +328,9 @@ class MaxRecordsPerUserTest(parameterized.TestCase): def test_histogram_noise_scales_exactly_with_k(self): k = 4 counts = np.array([10.0, 20.0, 30.0]) - base = primitives.DPGaussianHistogram(domain_size=3).configure(zcdp_rho=1.0) - scaled = primitives.DPGaussianHistogram( - domain_size=3, max_records_per_user=k - ).configure(zcdp_rho=1.0) + config = primitives.DPGaussianHistogramConfig(domain_size=3) + base = config.configure(zcdp_rho=1.0) + scaled = config.configure(zcdp_rho=1.0, max_records_per_user=k) base_noise = base(np.random.default_rng(0), counts).counts - counts scaled_noise = scaled(np.random.default_rng(0), counts).counts - counts np.testing.assert_allclose(scaled_noise, k * base_noise) @@ -336,9 +338,9 @@ def test_histogram_noise_scales_exactly_with_k(self): def test_count_noise_scales_exactly_with_k(self): k = 4 - base = primitives.DPGaussianCount().configure(zcdp_rho=1.0) - scaled = primitives.DPGaussianCount(max_records_per_user=k).configure( - zcdp_rho=1.0 + base = primitives.DPGaussianCountConfig().configure(zcdp_rho=1.0) + scaled = primitives.DPGaussianCountConfig().configure( + zcdp_rho=1.0, max_records_per_user=k ) base_noise = base.noisy_count(np.random.default_rng(0), 100) - 100 scaled_noise = scaled.noisy_count(np.random.default_rng(0), 100) - 100 @@ -346,20 +348,18 @@ def test_count_noise_scales_exactly_with_k(self): self.assertEqual(repr(scaled.dp_event), repr(base.dp_event)) def test_quantiles_accounting_invariant_to_k(self): - base = primitives.DPQuantiles( + config = primitives.DPQuantilesConfig( num_partitions=4, lower=0.0, upper=10.0 - ).configure(zcdp_rho=1.0) - scaled = primitives.DPQuantiles( - num_partitions=4, lower=0.0, upper=10.0, max_records_per_user=4 - ).configure(zcdp_rho=1.0) + ) + base = config.configure(zcdp_rho=1.0) + scaled = config.configure(zcdp_rho=1.0, max_records_per_user=4) self.assertEqual(repr(scaled.dp_event), repr(base.dp_event)) self.assertAlmostEqual(scaled.zcdp_rho, base.zcdp_rho) def test_partition_selection_accounting_invariant_to_k(self): - base = primitives.DPPartitionSelection(delta=1e-5).configure(zcdp_rho=1.0) - scaled = primitives.DPPartitionSelection( - delta=1e-5, max_records_per_user=4 - ).configure(zcdp_rho=1.0) + config = primitives.DPPartitionSelectionConfig(delta=1e-5) + base = config.configure(zcdp_rho=1.0) + scaled = config.configure(zcdp_rho=1.0, max_records_per_user=4) self.assertEqual(repr(scaled.dp_event), repr(base.dp_event)) def test_partition_selection_noise_scales_with_k(self): @@ -380,15 +380,21 @@ def test_partition_selection_noise_scales_with_k(self): @parameterized.named_parameters(("zero", 0), ("negative", -3)) def test_invalid_k_raises(self, k): with self.assertRaises(ValueError): - primitives.DPGaussianHistogram(domain_size=3, max_records_per_user=k) - with self.assertRaises(ValueError): - primitives.DPGaussianCount(max_records_per_user=k) + primitives.DPGaussianHistogramConfig(domain_size=3).configure( + zcdp_rho=1.0, max_records_per_user=k + ) with self.assertRaises(ValueError): - primitives.DPQuantiles( - num_partitions=4, lower=0.0, upper=10.0, max_records_per_user=k + primitives.DPGaussianCountConfig().configure( + zcdp_rho=1.0, max_records_per_user=k ) with self.assertRaises(ValueError): - primitives.DPPartitionSelection(delta=1e-5, max_records_per_user=k) + primitives.DPQuantilesConfig( + num_partitions=4, lower=0.0, upper=10.0 + ).configure(zcdp_rho=1.0, max_records_per_user=k) + with self.assertRaises(ValueError): + primitives.DPPartitionSelectionConfig(delta=1e-5).configure( + zcdp_rho=1.0, max_records_per_user=k + ) if __name__ == "__main__":