From 87b7ef2af0a40492f53ce2a859846df2582636df Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Fri, 3 Jul 2026 10:18:28 -0400 Subject: [PATCH 1/3] =?UTF-8?q?feat(hooks):=20active-surfacing=20proactive?= =?UTF-8?q?=20nudges=20(#97)=20=E2=80=94=20governance=20layer=20+=20comman?= =?UTF-8?q?d-cluster=20nudge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- stackunderflow/hooks/proactive.py | 800 +++++++++++++++++++ stackunderflow/hooks/recall.py | 82 +- stackunderflow/ingest/__init__.py | 12 + stackunderflow/settings.py | 23 + tests/stackunderflow/hooks/test_proactive.py | 500 ++++++++++++ 5 files changed, 1395 insertions(+), 22 deletions(-) create mode 100644 stackunderflow/hooks/proactive.py create mode 100644 tests/stackunderflow/hooks/test_proactive.py diff --git a/stackunderflow/hooks/proactive.py b/stackunderflow/hooks/proactive.py new file mode 100644 index 00000000..11df9e5a --- /dev/null +++ b/stackunderflow/hooks/proactive.py @@ -0,0 +1,800 @@ +"""Proactive nudge governance + the command-cluster nudge (spec 27 / #97). + +Where :mod:`stackunderflow.hooks.recall` decides *what* the memory store knows +about the thing a tool is about to touch, this module decides *whether that is +worth saying* — the anti-annoyance contract that the shipped hooks lack. It is +the single deterministic gate for every proactive/recall nudge: + +* **Governance** — a pure :func:`should_surface` ``(signal, state) -> bool`` and + a stateful :func:`admit` that records a fire. Enforces the §4 contract: + per-type allowlist, relevance floor, per-session dedupe by + ``sha1(type:target_key:signal_bucket)``, a global per-session cap, a + cross-session cooldown, and dismiss-driven adaptive quieting — all backed by a + small JSON file at ``~/.stackunderflow/proactive_state.json`` (file-locked, + bounded, corrupt/missing → treated as empty). **Never** ``store.db`` — hooks + must not contend with the ingest writer on the hot path. + +* **Phase 0 (governance retrofit)** — :func:`admit_file_risk` wraps the existing + ``recall.py`` file-risk output so a chronically risky file no longer nags + every session with no throttle. + +* **Phase 1 (command-cluster nudge)** — :func:`command_cluster_block` extracts a + pending Bash command's normalised head (via ``patterns._normalise_command``, + reused verbatim for key parity), looks it up in a precomputed O(1) signal + cache (``~/.stackunderflow/proactive_signals.json``, refreshed on ingest by + :func:`refresh_signal_cache`), applies the relevance floor + governance, and + renders one deterministic advisory line. + +Invariants (this runs inside users' live sessions — non-negotiable): + +* **Opt-in, off by default.** ``proactive_enabled`` defaults false. When it is + off the module is inert (:func:`mode` → ``"passthrough"``): ``recall.py`` + keeps its shipped, ungoverned behavior and no state file is written. The env + kill-switch ``STACKUNDERFLOW_PROACTIVE_DISABLED=1`` (:func:`mode` → ``"off"``) + silences everything and wins over ``proactive_enabled``. +* **Never blocks, never raises, always exit 0.** Nothing here ever returns a + PreToolUse deny/ask decision — only advisory ``additionalContext`` text is + ever produced by the callers. Any error, missing/corrupt state, or lock + contention degrades to "silent" (empty / ``False``), never to spam. +* **Fast + local.** No LLM, no network, no ``store.db`` write on the hook path. + The command lookup is an O(1) dict read against the precomputed cache — never + a live ``mine_patterns`` scan. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import time +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING, Any, Iterator + +from stackunderflow.hooks.inject import _slug_from_cwd + +if TYPE_CHECKING: # pragma: no cover - import only for the type annotation + import sqlite3 + +logger = logging.getLogger("stackunderflow.hooks") + +# ── filenames / knobs ─────────────────────────────────────────────────────── + +# Governance state + the precomputed signal cache both live in the app dir +# (derived from ``deps.store_path`` so a test that relocates the store relocates +# these too). JSON files, never the DB. +_STATE_FILENAME = "proactive_state.json" +_SIGNAL_FILENAME = "proactive_signals.json" +_LOCK_SUFFIX = ".lock" + +# Hard env kill-switch — wins over ``proactive_enabled`` and every other knob. +_KILL_SWITCH_ENV = "STACKUNDERFLOW_PROACTIVE_DISABLED" + +# The nudge type ids this module understands (mirrors ``proactive_types``). +TYPE_COMMAND_CLUSTER = "command-cluster" +TYPE_FILE_RISK = "file-risk" +_KNOWN_TYPES = frozenset({TYPE_COMMAND_CLUSTER, TYPE_FILE_RISK}) + +# Relevance floor: a cluster's last failure must be at most this many days old +# for the nudge to be "in the moment". Mirrors ``patterns.DEFAULT_SINCE_DAYS`` +# (the mining window) — a soft guard so a *stale* cache can't nudge on ancient +# failures. Kept as a local constant to keep the hot path free of a +# ``patterns`` import for non-Bash fires. +_RECENT_DAYS = 90 + +# Bounds so neither JSON file can grow without limit (LRU-style eviction). +_MAX_SESSIONS = 256 +_MAX_COOLDOWNS = 1024 +_MAX_FEEDBACK = 1024 +_MAX_PROJECTS_CACHED = 128 +_MAX_CLUSTERS_PER_PROJECT = 200 +_MAX_FILE_RISK_PER_PROJECT = 200 + +# File-lock acquisition budget. A short spin, then a stale-lock breaker — a hook +# must never wedge on a leaked lock. +_LOCK_TIMEOUT_S = 1.0 +_LOCK_SPIN_S = 0.01 +_LOCK_STALE_S = 10.0 + +# Rendered command-cluster block is one line — capped defensively. +_CMD_MAX_CHARS = 600 + +_SIGNAL_CACHE_VERSION = 1 + + +# ── config snapshot / mode ────────────────────────────────────────────────── + + +def _kill_switch() -> bool: + """True when the hard env kill-switch is set (wins over everything).""" + return os.environ.get(_KILL_SWITCH_ENV, "").strip().lower() in ("1", "true", "yes", "on") + + +@dataclass(frozen=True) +class Policy: + """Resolved governance config for one decision — env > file > default.""" + + enabled: bool + kill_switch: bool + types: frozenset[str] + max_per_session: int + cooldown_hours: float + dismiss_suppress_after: int + + @property + def mode(self) -> str: + """``"off"`` (kill-switch) · ``"passthrough"`` (disabled) · ``"governed"``.""" + if self.kill_switch: + return "off" + return "governed" if self.enabled else "passthrough" + + @classmethod + def from_settings(cls) -> Policy: + import stackunderflow.deps as deps + + cfg = deps.config + return cls( + enabled=bool(cfg.get("proactive_enabled")), + kill_switch=_kill_switch(), + types=_parse_types(cfg.get("proactive_types")), + max_per_session=_as_int(cfg.get("proactive_max_per_session"), 3), + cooldown_hours=_as_float(cfg.get("proactive_cooldown_hours"), 24.0), + dismiss_suppress_after=_as_int(cfg.get("proactive_dismiss_suppress_after"), 3), + ) + + +def mode() -> str: + """Current surfacing mode without building a full :class:`Policy`. + + ``"off"`` — kill-switch set; silence every pre-tool nudge. + ``"passthrough"`` — proactive disabled (default); ``recall.py`` keeps its + shipped ungoverned behavior, no new nudge types, no state writes. + ``"governed"`` — opt-in on; governance + the command-cluster nudge are live. + """ + if _kill_switch(): + return "off" + try: + import stackunderflow.deps as deps + + return "governed" if bool(deps.config.get("proactive_enabled")) else "passthrough" + except Exception: # noqa: BLE001 - a config read must never break the hook + return "passthrough" + + +def _parse_types(raw: Any) -> frozenset[str]: + """Parse the ``proactive_types`` allowlist leniently into known type ids.""" + if not isinstance(raw, str): + return frozenset(_KNOWN_TYPES) + out = {t.strip().lower() for t in raw.split(",") if t.strip()} + return frozenset(out & _KNOWN_TYPES) + + +def _as_int(value: Any, default: int) -> int: + try: + if isinstance(value, bool): + return default + return int(value) + except (TypeError, ValueError): + return default + + +def _as_float(value: Any, default: float) -> float: + try: + if isinstance(value, bool): + return default + return float(value) + except (TypeError, ValueError): + return default + + +# ── the signal ────────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class Signal: + """One would-be nudge, reduced to what governance needs to decide. + + ``counts`` are the two salient integers whose *coarse bucket* forms the + ``signal_bucket`` half of the fingerprint, so a materially worse situation + (counts crossing into a higher bucket) re-arms an already-fired nudge. + ``eligible`` carries the type-specific relevance-floor result. + """ + + type: str + target_key: str + session_id: str + counts: tuple[int, int] + eligible: bool + + @property + def bucket(self) -> str: + return f"{_coarse(self.counts[0])}.{_coarse(self.counts[1])}" + + @property + def fingerprint(self) -> str: + raw = f"{self.type}:{self.target_key}:{self.bucket}" + return hashlib.sha1(raw.encode("utf-8", "replace")).hexdigest() # noqa: S324 - dedupe key, not a security digest + + +def make_signal( + sig_type: str, + target_key: str, + session_id: str | None, + counts: tuple[int, int], + *, + eligible: bool, +) -> Signal: + return Signal( + type=sig_type, + target_key=str(target_key), + session_id=session_id or "", + counts=(int(counts[0]), int(counts[1])), + eligible=bool(eligible), + ) + + +def _coarse(n: int) -> int: + """Monotonic coarse tier for a count — 0,1,{2-4},{5-9},{10-49},{50+}.""" + n = max(0, int(n)) + if n <= 1: + return n + if n <= 4: + return 2 + if n <= 9: + return 3 + if n <= 49: + return 4 + return 5 + + +# ── the gate (pure) ───────────────────────────────────────────────────────── + + +def should_surface( + signal: Signal, + state: dict, + *, + policy: Policy | None = None, + now: datetime | None = None, +) -> bool: + """Deterministic gate — may this nudge surface, given *state*? Pure, no I/O. + + An LLM decides nothing here (spec §4.7). Order is cheapest-reject-first: + mode → type allowlist → relevance floor → adaptive quieting → per-session + dedupe → cooldown → frequency cap. Any doubt resolves to ``False``. + """ + policy = policy or Policy.from_settings() + now = now or _utcnow() + + if policy.mode != "governed": + return False + if signal.type not in policy.types: + return False + if not signal.eligible: + return False + + feedback = state.get("feedback") if isinstance(state.get("feedback"), dict) else {} + threshold = policy.dismiss_suppress_after + if threshold > 0 and ( + _dismissed(feedback, signal.type) >= threshold + or _dismissed(feedback, signal.fingerprint) >= threshold + ): + return False # adaptive quieting — the user keeps dismissing this + + sessions = state.get("sessions") if isinstance(state.get("sessions"), dict) else {} + sess = sessions.get(signal.session_id) if isinstance(sessions.get(signal.session_id), dict) else {} + + fired = sess.get("fired") + if isinstance(fired, list) and signal.fingerprint in fired: + return False # per-session dedupe + + cooldowns = state.get("cooldowns") if isinstance(state.get("cooldowns"), dict) else {} + until = _parse_iso(cooldowns.get(signal.fingerprint)) + if until is not None and until > now: + return False # cross-session cooldown + + if _as_int(sess.get("count"), 0) >= policy.max_per_session: + return False # frequency cap + + return True + + +def _dismissed(feedback: dict, key: str) -> int: + entry = feedback.get(key) + if isinstance(entry, dict): + return _as_int(entry.get("dismissed"), 0) + return 0 + + +# ── the gate (stateful) ───────────────────────────────────────────────────── + + +def admit(signal: Signal, *, now: datetime | None = None, policy: Policy | None = None) -> bool: + """Try to surface *signal*: check :func:`should_surface`, and on success + record the fire (dedupe set, count, cooldown, shown counter) to disk. + + Returns True only when the nudge should be shown *and* the fire was + recorded. Never raises; lock contention / a bad state file → ``False`` + (silent), never a duplicate or a crash. + """ + policy = policy or Policy.from_settings() + now = now or _utcnow() + if policy.mode != "governed": + return False + if signal.type not in policy.types or not signal.eligible: + return False + try: + with _locked(_state_path()) as locked: + if not locked: + return False # contended — fail to silence, never double-fire + state = _read_state() + if state is None: + return False # corrupt state → fail to silence, never spam / raise + if not should_surface(signal, state, policy=policy, now=now): + return False + _record_fire(state, signal, policy, now) + _write_json(_state_path(), state) + return True + except Exception: # noqa: BLE001 - governance must never disrupt the agent + logger.debug("proactive.admit swallowed an error", exc_info=True) + return False + + +def admit_file_risk(recalls: list[dict], payload: dict, *, now: datetime | None = None) -> bool: + """Phase 0: govern the shipped ``recall.py`` file-risk finding. + + Fingerprinted on the primary (highest-risk) path with a bucket over + ``failed``/``reverted``. Called by ``recall.py`` only in governed mode; in + passthrough mode recall never routes here and keeps its shipped behavior. + """ + if not recalls: + return False + primary = recalls[0] + target = primary.get("path") or "" + failed = sum(_as_int(r.get("failed"), 0) for r in recalls) + reverted = sum(_as_int(r.get("reverted"), 0) for r in recalls) + signal = make_signal( + TYPE_FILE_RISK, + target, + _session_id(payload), + (failed, reverted), + eligible=(failed + reverted) >= 1, + ) + return admit(signal, now=now) + + +def _record_fire(state: dict, signal: Signal, policy: Policy, now: datetime) -> None: + """Mutate *state* to reflect that *signal* just fired (in-place).""" + sessions = state.setdefault("sessions", {}) + if not isinstance(sessions, dict): + sessions = state["sessions"] = {} + sess = sessions.get(signal.session_id) + if not isinstance(sess, dict): + sess = sessions[signal.session_id] = {"fired": [], "count": 0} + fired = sess.setdefault("fired", []) + if not isinstance(fired, list): + fired = sess["fired"] = [] + if signal.fingerprint not in fired: + fired.append(signal.fingerprint) + sess["count"] = _as_int(sess.get("count"), 0) + 1 + sess["ts"] = now.isoformat() + + if policy.cooldown_hours > 0: + cooldowns = state.setdefault("cooldowns", {}) + if isinstance(cooldowns, dict): + cooldowns[signal.fingerprint] = (now + timedelta(hours=policy.cooldown_hours)).isoformat() + + feedback = state.setdefault("feedback", {}) + if isinstance(feedback, dict): + _bump(feedback, signal.type, "shown") + _bump(feedback, signal.fingerprint, "shown") + + _prune_state(state, now) + + +def record_dismissal(key: str, *, now: datetime | None = None) -> None: + """Register a dashboard 'don't show this again' for a type or a fingerprint. + + The Tier-2 dismiss primitive (the retrospective panel calls this; not wired + to a route in the MVP). Increments the ``dismissed`` counter that + :func:`should_surface` reads for adaptive quieting. Never raises. + """ + now = now or _utcnow() + try: + with _locked(_state_path()) as locked: + if not locked: + return + state = _read_state() + if state is None: + state = {} # dashboard side — a corrupt file is safe to reset here + feedback = state.setdefault("feedback", {}) + if isinstance(feedback, dict): + _bump(feedback, str(key), "dismissed") + _prune_state(state, now) + _write_json(_state_path(), state) + except Exception: # noqa: BLE001 + logger.debug("proactive.record_dismissal swallowed an error", exc_info=True) + + +def _bump(feedback: dict, key: str, field_name: str) -> None: + entry = feedback.get(key) + if not isinstance(entry, dict): + entry = feedback[key] = {"shown": 0, "dismissed": 0} + entry[field_name] = _as_int(entry.get(field_name), 0) + 1 + + +# ── the command-cluster nudge (Phase 1) ───────────────────────────────────── + + +def command_cluster_block(payload: dict, *, now: datetime | None = None) -> str: + """Advisory line for a pending Bash command in a known failure cluster, or ``""``. + + O(1): normalise the command head and look it up in the precomputed cache; + apply the relevance floor (``failure_count ≥ 2`` and ``session_count ≥ 2`` + and recent) and governance. Never runs a live ``mine_patterns`` scan, never + raises. + """ + try: + if not isinstance(payload, dict) or payload.get("tool_name") != "Bash": + return "" + tool_input = payload.get("tool_input") + if not isinstance(tool_input, dict): + return "" + command = tool_input.get("command") + if not isinstance(command, str) or not command.strip(): + return "" + slug = _slug_from_cwd(payload.get("cwd")) + if not slug: + return "" + + from stackunderflow.reports.patterns import _normalise_command + + key = _normalise_command(command) # VERBATIM reuse — cluster-key parity + cluster = _lookup_cluster(slug, key) + if cluster is None: + return "" + + now = now or _utcnow() + failure_count = _as_int(cluster.get("failure_count"), 0) + session_count = _as_int(cluster.get("session_count"), 0) + eligible = ( + failure_count >= 2 + and session_count >= 2 + and _is_recent(cluster.get("last_failure_ts"), now) + ) + signal = make_signal( + TYPE_COMMAND_CLUSTER, key, _session_id(payload), (failure_count, session_count), eligible=eligible + ) + if not admit(signal, now=now): + return "" + return _render_command_cluster(cluster, key) + except Exception: # noqa: BLE001 - a nudge must never disrupt the agent + logger.debug("proactive.command_cluster_block swallowed an error", exc_info=True) + return "" + + +def _render_command_cluster(cluster: dict, key: str) -> str: + """One deterministic advisory line for a command-cluster nudge.""" + command = cluster.get("command") if isinstance(cluster.get("command"), str) else key + command = command or key + session_count = _as_int(cluster.get("session_count"), 0) + sess_word = "session" if session_count == 1 else "sessions" + text = ( + f"[StackUnderflow memory] Heads-up before this Bash call: `{command}` has failed in " + f"{session_count} recent {sess_word} in this project" + ) + top = _top_category(cluster.get("categories")) + if top: + text += f" — mostly {top}" + text += "." + date = cluster.get("last_failure_ts") + if isinstance(date, str) and date: + text += f" Last failure {date[:10]}." + if len(text) > _CMD_MAX_CHARS: + text = text[: max(1, _CMD_MAX_CHARS - 1)].rstrip() + "…" + return text + + +def _top_category(categories: Any) -> str | None: + if not isinstance(categories, dict) or not categories: + return None + try: + return max(categories.items(), key=lambda kv: (_as_int(kv[1], 0), str(kv[0])))[0] + except (TypeError, ValueError): + return None + + +def _lookup_cluster(slug: str, key: str) -> dict | None: + """O(1) read of one cluster from the precomputed cache; ``None`` if absent.""" + cache = _read_json(_signal_path()) + projects = cache.get("projects") + if not isinstance(projects, dict): + return None + entry = projects.get(slug) + if not isinstance(entry, dict): + return None + clusters = entry.get("command_clusters") + if not isinstance(clusters, dict): + return None + cluster = clusters.get(key) + return cluster if isinstance(cluster, dict) else None + + +# ── signal cache precompute (ingest side) ─────────────────────────────────── + + +def refresh_signal_cache(conn: "sqlite3.Connection", slugs: set[str] | list[str]) -> None: + """Recompute + persist the command/file signal cache for the given slugs. + + Called additively from the ingest/reindex path. **Self-gates on + ``proactive_enabled``** so the default (opt-out) path pays nothing — no + ``mine_patterns`` scan is run unless a user turned the feature on. Fenced: + any failure is swallowed so a cache hiccup can never break ingest. + """ + policy = Policy.from_settings() + if policy.mode != "governed": + return # opt-in only — no precompute cost for users who never enabled it + slug_list = [s for s in dict.fromkeys(slugs) if s] + if not slug_list: + return + try: + from stackunderflow.reports import patterns + from stackunderflow.store import queries + + now_iso = _utcnow().isoformat() + with _locked(_signal_path()) as locked: + if not locked: + return + cache = _read_json(_signal_path()) + projects = cache.get("projects") + if not isinstance(projects, dict): + projects = {} + for slug in slug_list: + ids = [row.id for row in queries.get_projects_by_slug(conn, slug=slug)] + if not ids: + continue + report = patterns.mine_patterns(conn, project_ids=ids) + projects[slug] = { + "generated_at": now_iso, + "command_clusters": _clusters_map(report.get("command_clusters")), + "file_risk": _file_risk_map(report.get("file_risk")), + } + cache = { + "version": _SIGNAL_CACHE_VERSION, + "generated_at": now_iso, + "projects": _cap_projects(projects), + } + _write_json(_signal_path(), cache) + except Exception: # noqa: BLE001 - a signal-cache refresh must never break ingest + logger.debug("proactive.refresh_signal_cache swallowed an error", exc_info=True) + + +def _clusters_map(clusters: Any) -> dict: + """``command_clusters`` list → ``{normalised_key: trimmed cluster}`` (O(1) lookup).""" + out: dict[str, dict] = {} + if not isinstance(clusters, list): + return out + for c in clusters[:_MAX_CLUSTERS_PER_PROJECT]: + if not isinstance(c, dict): + continue + key = c.get("command") + if not isinstance(key, str) or not key: + continue + cats = c.get("categories") + out[key] = { + "command": key, + "failure_count": _as_int(c.get("failure_count"), 0), + "session_count": _as_int(c.get("session_count"), 0), + "categories": cats if isinstance(cats, dict) else {}, + "last_failure_ts": c.get("last_failure_ts") if isinstance(c.get("last_failure_ts"), str) else None, + } + return out + + +def _file_risk_map(file_risk: Any) -> dict: + """``file_risk`` list → ``{path: trimmed risk}``. Cached for Tier-2 / Phase 2; + the MVP hook path does not read it (file-risk stays on the recall CLI path).""" + out: dict[str, dict] = {} + if not isinstance(file_risk, list): + return out + for f in file_risk[:_MAX_FILE_RISK_PER_PROJECT]: + if not isinstance(f, dict): + continue + path = f.get("path") + if not isinstance(path, str) or not path: + continue + out[path] = { + "failure_count": _as_int(f.get("failure_count"), 0), + "failure_session_count": _as_int(f.get("failure_session_count"), 0), + "last_failure_ts": f.get("last_failure_ts") if isinstance(f.get("last_failure_ts"), str) else None, + } + return out + + +def _cap_projects(projects: dict) -> dict: + """Bound the cache to the most-recently-generated projects.""" + if len(projects) <= _MAX_PROJECTS_CACHED: + return projects + ordered = sorted( + projects.items(), key=lambda kv: str(kv[1].get("generated_at", "")), reverse=True + ) + return dict(ordered[:_MAX_PROJECTS_CACHED]) + + +# ── state pruning (bounded LRU) ───────────────────────────────────────────── + + +def _prune_state(state: dict, now: datetime) -> None: + """Keep the state file bounded — evict old sessions, expired cooldowns.""" + sessions = state.get("sessions") + if isinstance(sessions, dict) and len(sessions) > _MAX_SESSIONS: + ordered = sorted(sessions.items(), key=lambda kv: str(kv[1].get("ts", "")), reverse=True) + state["sessions"] = dict(ordered[:_MAX_SESSIONS]) + + cooldowns = state.get("cooldowns") + if isinstance(cooldowns, dict): + live = { + fp: ts + for fp, ts in cooldowns.items() + if (parsed := _parse_iso(ts)) is not None and parsed > now + } + if len(live) > _MAX_COOLDOWNS: + ordered = sorted(live.items(), key=lambda kv: str(kv[1]), reverse=True) + live = dict(ordered[:_MAX_COOLDOWNS]) + state["cooldowns"] = live + + feedback = state.get("feedback") + if isinstance(feedback, dict) and len(feedback) > _MAX_FEEDBACK: + ordered = sorted( + feedback.items(), + key=lambda kv: (_as_int(kv[1].get("dismissed"), 0), _as_int(kv[1].get("shown"), 0)) + if isinstance(kv[1], dict) + else (0, 0), + reverse=True, + ) + state["feedback"] = dict(ordered[:_MAX_FEEDBACK]) + + +# ── paths / JSON I/O / file lock ──────────────────────────────────────────── + + +def _app_dir() -> Path: + """The app dir — derived from ``deps.store_path`` so tests relocate cleanly.""" + import stackunderflow.deps as deps + + return deps.store_path.parent + + +def _state_path() -> Path: + return _app_dir() / _STATE_FILENAME + + +def _signal_path() -> Path: + return _app_dir() / _SIGNAL_FILENAME + + +def _read_json(path: Path) -> dict: + """Load a JSON dict; missing / corrupt / non-dict → ``{}`` (fail to empty).""" + try: + if not path.exists(): + return {} + data = json.loads(path.read_text()) + except (OSError, ValueError, TypeError): + return {} + return data if isinstance(data, dict) else {} + + +def _read_state() -> dict | None: + """Governance state, distinguishing *missing* from *corrupt*. + + * **Missing** file → ``{}`` — the normal first-fire condition; an empty + state suppresses nothing, so the nudge proceeds under the usual rules. + * **Corrupt** / non-dict → ``None`` — an error the hot path resolves by + failing to silence (never spam off unreadable throttle state), never a + raise. (Writes go through ``os.replace``, so corruption is not expected + in practice.) + """ + path = _state_path() + try: + if not path.exists(): + return {} + data = json.loads(path.read_text()) + except (OSError, ValueError, TypeError): + return None + return data if isinstance(data, dict) else None + + +def _write_json(path: Path, data: dict) -> None: + """Atomically persist *data* (temp file + ``os.replace``). Best-effort.""" + try: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + f".tmp-{os.getpid()}") + tmp.write_text(json.dumps(data)) + os.replace(tmp, path) + except OSError: + logger.debug("proactive: could not write %s", path, exc_info=True) + + +@contextmanager +def _locked(target: Path) -> Iterator[bool]: + """Best-effort cross-process advisory lock for a read-modify-write on *target*. + + Uses an ``O_CREAT|O_EXCL`` sibling lock file — portable across platforms + (no ``fcntl``). Yields True when acquired, False on timeout (the caller + then bails to silence). A lock older than ``_LOCK_STALE_S`` is treated as + leaked and stolen, so a crashed hook can never wedge the feature. + """ + lock_path = target.with_suffix(target.suffix + _LOCK_SUFFIX) + try: + lock_path.parent.mkdir(parents=True, exist_ok=True) + except OSError: + yield False + return + deadline = time.monotonic() + _LOCK_TIMEOUT_S + fd: int | None = None + while True: + try: + fd = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + break + except FileExistsError: + try: + if time.time() - os.path.getmtime(lock_path) > _LOCK_STALE_S: + os.unlink(lock_path) + continue + except OSError: + pass + if time.monotonic() >= deadline: + yield False + return + time.sleep(_LOCK_SPIN_S) + except OSError: + yield False + return + try: + yield True + finally: + if fd is not None: + try: + os.close(fd) + except OSError: + pass + try: + os.unlink(lock_path) + except OSError: + pass + + +# ── small utils ───────────────────────────────────────────────────────────── + + +def _session_id(payload: dict) -> str: + sid = payload.get("session_id") if isinstance(payload, dict) else None + return sid if isinstance(sid, str) and sid else "" + + +def _utcnow() -> datetime: + return datetime.now(UTC) + + +def _parse_iso(value: Any) -> datetime | None: + if not isinstance(value, str) or not value: + return None + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + except (ValueError, TypeError): + return None + return dt if dt.tzinfo is not None else dt.replace(tzinfo=UTC) + + +def _is_recent(ts: Any, now: datetime) -> bool: + """True when *ts* is a parseable timestamp within ``_RECENT_DAYS`` of *now*. + + A missing / unparseable timestamp is *not* recent — a nudge without a + dateable last failure stays silent (conservative).""" + parsed = _parse_iso(ts) + if parsed is None: + return False + return (now - parsed) <= timedelta(days=_RECENT_DAYS) diff --git a/stackunderflow/hooks/recall.py b/stackunderflow/hooks/recall.py index 4ba578f1..605760bb 100644 --- a/stackunderflow/hooks/recall.py +++ b/stackunderflow/hooks/recall.py @@ -52,7 +52,7 @@ import time from typing import Any -from stackunderflow.hooks import templates +from stackunderflow.hooks import proactive, templates logger = logging.getLogger("stackunderflow.hooks") @@ -97,6 +97,15 @@ def build_recall(hook_id: str, payload: dict | None) -> str: Never raises. Any failure — unknown id, bad payload, missing CLI, timeout, garbage output — returns ``""`` so the caller emits nothing and exits 0. An empty return is also the normal "file is clean" outcome. + + Governance (spec 27 / #97) rides on top without changing the default: + + * ``proactive`` disabled (the default) → **passthrough**: the shipped + file-risk warning is emitted exactly as before, ungoverned. + * kill-switch set → **off**: every pre-tool nudge is silenced. + * ``proactive_enabled`` → **governed**: the file-risk warning passes through + the dedupe / cap / cooldown layer (Phase 0), and a command-cluster nudge + (Phase 1) may be appended on the Bash path. """ try: payload = payload if isinstance(payload, dict) else {} @@ -104,35 +113,64 @@ def build_recall(hook_id: str, payload: dict | None) -> str: if event is None or hook_id not in templates.RECALL_HOOK_IDS: return "" - paths = _candidate_paths(payload) - if not paths: - return "" + pmode = proactive.mode() + if pmode == "off": + return "" # env kill-switch — silence every pre-tool nudge + + blocks: list[str] = [] - cwd = payload.get("cwd") - cwd = cwd if isinstance(cwd, str) and os.path.isdir(cwd) else None - - deadline = time.monotonic() + _timeout_seconds() - recalls: list[dict] = [] - for path in paths: - remaining = deadline - time.monotonic() - if remaining <= 0.05: - break # deadline spent — never stretch it for more paths - envelope = _query_memory_file(path, timeout=remaining, cwd=cwd) - if envelope is None: - continue - recall = _extract_recall(envelope, path) - if recall is not None: - recalls.append(recall) - - text = _render(recalls) - if not text.strip(): + # ── file-risk (shipped in #5; #97 only retrofits governance) ────────── + recalls = _collect_recalls(payload) + file_text = _render(recalls) + if file_text.strip(): + if pmode != "governed" or proactive.admit_file_risk(recalls, payload): + blocks.append(file_text) + + # ── command-cluster nudge (Phase 1 — governed mode, Bash path only) ─── + if pmode == "governed": + cmd_text = proactive.command_cluster_block(payload) + if cmd_text.strip(): + blocks.append(cmd_text) + + if not blocks: return "" + text = "\n\n".join(blocks) return json.dumps({"hookSpecificOutput": {"hookEventName": event, "additionalContext": text}}) except Exception: # noqa: BLE001 - a recall hook must never disrupt the agent logger.debug("recall hook %s swallowed an error", hook_id, exc_info=True) return "" +def _collect_recalls(payload: dict) -> list[dict]: + """Run the file-risk lookups for a fire and return the risk findings. + + The path-extraction + shared-deadline CLI loop, factored out of + :func:`build_recall` so the findings can be handed to the governance layer + before rendering. Empty list when there is nothing to look up (no + extractable path) or nothing risky came back. + """ + paths = _candidate_paths(payload) + if not paths: + return [] + + cwd = payload.get("cwd") + cwd = cwd if isinstance(cwd, str) and os.path.isdir(cwd) else None + + deadline = time.monotonic() + _timeout_seconds() + recalls: list[dict] = [] + for path in paths: + remaining = deadline - time.monotonic() + if remaining <= 0.05: + break # deadline spent — never stretch it for more paths + envelope = _query_memory_file(path, timeout=remaining, cwd=cwd) + if envelope is None: + continue + recall = _extract_recall(envelope, path) + if recall is not None: + recalls.append(recall) + return recalls + + # ── payload → candidate paths ─────────────────────────────────────────────── diff --git a/stackunderflow/ingest/__init__.py b/stackunderflow/ingest/__init__.py index a2a3cd3e..52d6de79 100644 --- a/stackunderflow/ingest/__init__.py +++ b/stackunderflow/ingest/__init__.py @@ -155,6 +155,18 @@ def auto_reindex_touched( finally: deps.is_reindexing = prior_flag + # Proactive signal cache (spec 27 / #97): precompute the O(1) command / + # file signal snapshot that the pre-tool hook reads, so the hook never runs + # a live pattern scan. Self-gates on ``proactive_enabled`` (zero cost when + # the feature is off) and swallows its own errors — additive, never blocks + # ingest. + try: + from stackunderflow.hooks import proactive + + proactive.refresh_signal_cache(conn, slug_list) + except Exception as e: # noqa: BLE001 — signal precompute must never break ingest + _logger.debug("proactive signal-cache refresh skipped: %s", e) + def _lookup(adapters: list[SourceAdapter], name: str) -> SourceAdapter: for a in adapters: diff --git a/stackunderflow/settings.py b/stackunderflow/settings.py index 25ce4033..b152baf3 100644 --- a/stackunderflow/settings.py +++ b/stackunderflow/settings.py @@ -161,6 +161,29 @@ class Settings: # the wrong number of components falls back to the default. A future # ``cite_rate`` term (citation-feedback spec) appends a fourth weight. discovery_rank_weights = _Opt("0.5,0.2,0.3", "STACKUNDERFLOW_DISCOVERY_RANK_WEIGHTS") + # ── Proactive surfacing (spec 27 / #97) — the anti-annoyance governance + # knobs for the pre-tool nudge surface (:mod:`stackunderflow.hooks.proactive`). + # OPT-IN: ``proactive_enabled`` is false by default, so the retrofitted + # recall governance and the command-cluster nudge stay dormant until a + # user turns them on. A hard env kill-switch + # ``STACKUNDERFLOW_PROACTIVE_DISABLED=1`` (read directly in the hook, not a + # setting here) wins over every one of these. + proactive_enabled = _Opt(False, "STACKUNDERFLOW_PROACTIVE_ENABLED") + # Per-type allowlist — comma-separated nudge type ids. Only listed types may + # surface. Env-settable so a shell can flip it fast on the hook path. + proactive_types = _Opt("command-cluster,file-risk", + "STACKUNDERFLOW_PROACTIVE_TYPES") + # Frequency cap: at most this many nudges per Claude Code session, global + # across all nudge types. Cap reached → silent. + proactive_max_per_session = _Opt(3, "STACKUNDERFLOW_PROACTIVE_MAX_PER_SESSION") + # Cross-session cooldown: once a nudge fingerprint fires, it stays quiet for + # this many hours even across sessions (a chronically risky target doesn't + # nag every session). + proactive_cooldown_hours = _Opt(24, "STACKUNDERFLOW_PROACTIVE_COOLDOWN_HOURS") + # Adaptive quieting: after this many dashboard dismissals of a type (or a + # specific fingerprint) it is auto-suppressed. File-only — a dashboard-side + # tuning knob, not a hot-path env read. + proactive_dismiss_suppress_after = _Opt(3, None) # ── public helpers (used by server.py / cli.py) ────────────────────── diff --git a/tests/stackunderflow/hooks/test_proactive.py b/tests/stackunderflow/hooks/test_proactive.py new file mode 100644 index 00000000..07a1f91b --- /dev/null +++ b/tests/stackunderflow/hooks/test_proactive.py @@ -0,0 +1,500 @@ +"""Proactive nudge governance + command-cluster nudge (spec 27 / #97). + +The anti-annoyance contract, locked as tests. Everything seeds a synthetic +governance file / signal cache (or a tiny real store for the ingest +integration) and asserts exact behavior — no LLM, no network, an injected +clock, so nothing is wall-clock-flaky. + +Covered (the §10 matrix): + +* Signal → surface: fires on a seeded 3-failure command; silent on + 1-failure / clean / unknown; file-risk parity. +* Governance: per-session dedupe, global frequency cap, cross-session + cooldown (injected clock), dismiss-driven adaptive quieting (by type AND + by fingerprint), opt-out precedence (``proactive_enabled=false`` and the + env kill-switch). +* Invariants: corrupt state → silent, no raise; never a deny/ask decision, + only ``additionalContext``; ``_normalise_command`` key parity; token budget. +* Wiring: ``recall.py`` passthrough unchanged when disabled, governed when on + (Phase 0); ``refresh_signal_cache`` builds the O(1) snapshot the hook reads. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta + +import pytest + +import stackunderflow.deps as deps +from stackunderflow import settings as settings_mod +from stackunderflow.hooks import proactive, recall +from stackunderflow.reports.patterns import _normalise_command + +RECALL_ID = "stackunderflow-pretool-recall" + +# A pinned clock for the deterministic gate tests. The ingest-integration test +# seeds relative to the real clock (it drives the real ``mine_patterns`` window). +NOW = datetime(2026, 6, 30, 12, 0, 0, tzinfo=UTC) + + +# ── isolation ──────────────────────────────────────────────────────────────── + + +@pytest.fixture +def isolate(tmp_path, monkeypatch): + """Relocate the store dir + settings file to tmp; clear proactive env. + + ``proactive`` derives its state/cache paths from ``deps.store_path.parent``, + so pointing the store at tmp lands ``proactive_state.json`` / + ``proactive_signals.json`` there too. Settings are pinned to a tmp config + file so the developer's real config can't leak in. + """ + monkeypatch.setattr(deps, "store_path", tmp_path / "store.db") + monkeypatch.setattr(settings_mod, "_CFG_FILE", tmp_path / "config.json") + for env in ( + "STACKUNDERFLOW_PROACTIVE_DISABLED", + "STACKUNDERFLOW_PROACTIVE_ENABLED", + "STACKUNDERFLOW_PROACTIVE_TYPES", + "STACKUNDERFLOW_PROACTIVE_MAX_PER_SESSION", + "STACKUNDERFLOW_PROACTIVE_COOLDOWN_HOURS", + ): + monkeypatch.delenv(env, raising=False) + return tmp_path + + +def _enable(monkeypatch): + monkeypatch.setenv("STACKUNDERFLOW_PROACTIVE_ENABLED", "1") + + +# ── helpers ────────────────────────────────────────────────────────────────── + + +def _policy(**over): + base = dict( + enabled=True, + kill_switch=False, + types=frozenset({proactive.TYPE_COMMAND_CLUSTER, proactive.TYPE_FILE_RISK}), + max_per_session=3, + cooldown_hours=24.0, + dismiss_suppress_after=3, + ) + base.update(over) + return proactive.Policy(**base) + + +def _cmd_signal(key="npm install", session="s1", fc=3, sc=3, eligible=True): + return proactive.make_signal( + proactive.TYPE_COMMAND_CLUSTER, key, session, (fc, sc), eligible=eligible + ) + + +def _cluster(command="npm install", fc=3, sc=3, cats=None, last=None): + return { + "command": command, + "failure_count": fc, + "session_count": sc, + "categories": cats if cats is not None else {"Command Timeout": 3}, + "last_failure_ts": last if last is not None else (NOW - timedelta(days=2)).isoformat(), + } + + +def _write_cache(clusters, *, cwd="/repo/demo"): + slug = proactive._slug_from_cwd(cwd) + cache = { + "version": 1, + "generated_at": NOW.isoformat(), + "projects": {slug: {"generated_at": NOW.isoformat(), "command_clusters": clusters, "file_risk": {}}}, + } + proactive._write_json(proactive._signal_path(), cache) + return slug + + +def _bash(command, *, cwd="/repo/demo", session="s1"): + return { + "hook_event_name": "PreToolUse", + "tool_name": "Bash", + "tool_input": {"command": command}, + "cwd": cwd, + "session_id": session, + } + + +# ── (1) command normalisation parity — the lookup key ──────────────────────── + + +class TestNormalisationParity: + """The hook must key on the SAME string ``patterns`` clustered on.""" + + @pytest.mark.parametrize( + "pending", + ["npm install", "cd /repo && npm install", "NODE_ENV=prod npm install", "cd x && npm install --no-fund"], + ) + def test_pending_command_maps_to_cluster_key(self, isolate, monkeypatch, pending): + _enable(monkeypatch) + _write_cache({"npm install": _cluster()}) + out = proactive.command_cluster_block(_bash(pending), now=NOW) + assert "npm install" in out # normalised head matched the cached cluster + + def test_uses_patterns_normaliser_verbatim(self): + # Parity is by *reuse*, not a re-implementation. + assert _normalise_command("cd x && npm install") == "npm install" + assert _normalise_command("NODE_ENV=prod npm install") == "npm install" + + +# ── (2) signal → surface ───────────────────────────────────────────────────── + + +class TestCommandClusterSignal: + def test_fires_on_seeded_three_failure_command(self, isolate, monkeypatch): + _enable(monkeypatch) + _write_cache({"npm install": _cluster(fc=3, sc=3)}) + out = proactive.command_cluster_block(_bash("npm install --no-fund"), now=NOW) + assert out.startswith("[StackUnderflow memory]") + assert "`npm install`" in out + assert "3 recent sessions" in out + assert "Command Timeout" in out + assert "Last failure" in out + + def test_silent_on_single_failure(self, isolate, monkeypatch): + _enable(monkeypatch) + _write_cache({"npm install": _cluster(fc=1, sc=1)}) + assert proactive.command_cluster_block(_bash("npm install"), now=NOW) == "" + + def test_silent_when_only_one_session(self, isolate, monkeypatch): + # 2 failures but both in ONE session — below the session_count floor. + _enable(monkeypatch) + _write_cache({"npm install": _cluster(fc=2, sc=1)}) + assert proactive.command_cluster_block(_bash("npm install"), now=NOW) == "" + + def test_silent_on_stale_last_failure(self, isolate, monkeypatch): + _enable(monkeypatch) + old = (NOW - timedelta(days=400)).isoformat() + _write_cache({"npm install": _cluster(last=old)}) + assert proactive.command_cluster_block(_bash("npm install"), now=NOW) == "" + + def test_silent_on_unknown_command(self, isolate, monkeypatch): + _enable(monkeypatch) + _write_cache({"npm install": _cluster()}) + assert proactive.command_cluster_block(_bash("pytest tests/ -q"), now=NOW) == "" + + def test_silent_on_empty_cache(self, isolate, monkeypatch): + _enable(monkeypatch) + _write_cache({}) + assert proactive.command_cluster_block(_bash("npm install"), now=NOW) == "" + + def test_silent_on_missing_cache(self, isolate, monkeypatch): + _enable(monkeypatch) # no cache file at all + assert proactive.command_cluster_block(_bash("npm install"), now=NOW) == "" + + def test_silent_on_non_bash(self, isolate, monkeypatch): + _enable(monkeypatch) + _write_cache({"npm install": _cluster()}) + payload = {"tool_name": "Edit", "tool_input": {"file_path": "/repo/x.py"}, "cwd": "/repo/demo"} + assert proactive.command_cluster_block(payload, now=NOW) == "" + + +# ── (3) the pure gate: should_surface ──────────────────────────────────────── + + +class TestShouldSurface: + def test_admits_a_fresh_eligible_signal(self): + assert proactive.should_surface(_cmd_signal(), {}, policy=_policy(), now=NOW) is True + + def test_rejects_ineligible(self): + assert proactive.should_surface(_cmd_signal(eligible=False), {}, policy=_policy(), now=NOW) is False + + def test_rejects_type_not_in_allowlist(self): + pol = _policy(types=frozenset({proactive.TYPE_FILE_RISK})) + assert proactive.should_surface(_cmd_signal(), {}, policy=pol, now=NOW) is False + + def test_dedupe_same_fingerprint_in_session(self): + sig = _cmd_signal() + state = {"sessions": {"s1": {"fired": [sig.fingerprint], "count": 1}}} + assert proactive.should_surface(sig, state, policy=_policy(), now=NOW) is False + + def test_cap_reached(self): + state = {"sessions": {"s1": {"fired": ["other"], "count": 1}}} + assert proactive.should_surface(_cmd_signal(), state, policy=_policy(max_per_session=1), now=NOW) is False + + def test_cooldown_blocks_then_expires(self): + sig = _cmd_signal() + until = (NOW + timedelta(hours=5)).isoformat() + state = {"cooldowns": {sig.fingerprint: until}} + assert proactive.should_surface(sig, state, policy=_policy(), now=NOW) is False + later = NOW + timedelta(hours=6) + assert proactive.should_surface(sig, state, policy=_policy(), now=later) is True + + def test_adaptive_quieting_by_type(self): + sig = _cmd_signal() + state = {"feedback": {proactive.TYPE_COMMAND_CLUSTER: {"shown": 9, "dismissed": 3}}} + assert proactive.should_surface(sig, state, policy=_policy(dismiss_suppress_after=3), now=NOW) is False + + def test_adaptive_quieting_by_fingerprint(self): + sig = _cmd_signal() + state = {"feedback": {sig.fingerprint: {"dismissed": 3}}} + assert proactive.should_surface(sig, state, policy=_policy(dismiss_suppress_after=3), now=NOW) is False + + def test_disabled_and_killswitch_reject(self): + assert proactive.should_surface(_cmd_signal(), {}, policy=_policy(enabled=False), now=NOW) is False + assert proactive.should_surface(_cmd_signal(), {}, policy=_policy(kill_switch=True), now=NOW) is False + + def test_worsening_counts_rearm_via_bucket(self): + # A fired fingerprint at counts (3,3); the same command now at (12,12) + # crosses into a higher bucket → a NEW fingerprint → not deduped. + low = _cmd_signal(fc=3, sc=3) + high = _cmd_signal(fc=12, sc=12) + assert low.fingerprint != high.fingerprint + state = {"sessions": {"s1": {"fired": [low.fingerprint], "count": 1}}} + assert proactive.should_surface(high, state, policy=_policy(), now=NOW) is True + + +# ── (4) the stateful gate: admit (end-to-end through the state file) ───────── + + +class TestAdmit: + def test_admit_then_dedupe(self, isolate, monkeypatch): + _enable(monkeypatch) + sig = _cmd_signal() + assert proactive.admit(sig, now=NOW) is True + assert proactive.admit(sig, now=NOW) is False # deduped (and cooling down) + + def test_admit_cap(self, isolate, monkeypatch): + _enable(monkeypatch) + monkeypatch.setenv("STACKUNDERFLOW_PROACTIVE_MAX_PER_SESSION", "1") + assert proactive.admit(_cmd_signal(key="npm install"), now=NOW) is True + assert proactive.admit(_cmd_signal(key="pytest"), now=NOW) is False # distinct fp, cap=1 + + def test_admit_cooldown_across_sessions(self, isolate, monkeypatch): + _enable(monkeypatch) + monkeypatch.setenv("STACKUNDERFLOW_PROACTIVE_COOLDOWN_HOURS", "24") + assert proactive.admit(_cmd_signal(session="s1"), now=NOW) is True + # Same command (→ same fingerprint) in a *different* session, still in cooldown. + assert proactive.admit(_cmd_signal(session="s2"), now=NOW + timedelta(hours=1)) is False + # After the cooldown window it re-arms. + assert proactive.admit(_cmd_signal(session="s2"), now=NOW + timedelta(hours=25)) is True + + def test_admit_suppressed_after_dismissals(self, isolate, monkeypatch): + _enable(monkeypatch) + for _ in range(3): # default proactive_dismiss_suppress_after = 3 + proactive.record_dismissal(proactive.TYPE_COMMAND_CLUSTER, now=NOW) + assert proactive.admit(_cmd_signal(), now=NOW) is False + + def test_admit_records_shown_counter(self, isolate, monkeypatch): + _enable(monkeypatch) + sig = _cmd_signal() + assert proactive.admit(sig, now=NOW) is True + state = json.loads(proactive._state_path().read_text()) + assert state["feedback"][proactive.TYPE_COMMAND_CLUSTER]["shown"] == 1 + assert sig.fingerprint in state["sessions"]["s1"]["fired"] + assert state["sessions"]["s1"]["count"] == 1 + assert sig.fingerprint in state["cooldowns"] + + +# ── (5) opt-out precedence ─────────────────────────────────────────────────── + + +class TestOptOut: + def test_disabled_is_silent(self, isolate, monkeypatch): + # proactive_enabled defaults false → passthrough → no command nudge. + _write_cache({"npm install": _cluster()}) + assert proactive.command_cluster_block(_bash("npm install"), now=NOW) == "" + + def test_killswitch_wins_even_when_enabled(self, isolate, monkeypatch): + _enable(monkeypatch) + monkeypatch.setenv("STACKUNDERFLOW_PROACTIVE_DISABLED", "1") + _write_cache({"npm install": _cluster()}) + assert proactive.mode() == "off" + assert proactive.command_cluster_block(_bash("npm install"), now=NOW) == "" + + def test_per_type_allowlist_excludes_command_cluster(self, isolate, monkeypatch): + _enable(monkeypatch) + monkeypatch.setenv("STACKUNDERFLOW_PROACTIVE_TYPES", "file-risk") + _write_cache({"npm install": _cluster()}) + assert proactive.command_cluster_block(_bash("npm install"), now=NOW) == "" + + +# ── (6) invariant guards ───────────────────────────────────────────────────── + + +class TestInvariants: + def test_corrupt_state_is_silent_no_raise(self, isolate, monkeypatch): + _enable(monkeypatch) + proactive._state_path().parent.mkdir(parents=True, exist_ok=True) + proactive._state_path().write_text("<<>>") + assert proactive.admit(_cmd_signal(), now=NOW) is False # fail to silence + + def test_missing_state_fires_first_time(self, isolate, monkeypatch): + # A *missing* state file is the normal first-fire condition — must fire. + _enable(monkeypatch) + assert not proactive._state_path().exists() + assert proactive.admit(_cmd_signal(), now=NOW) is True + + def test_command_block_never_emits_a_decision(self, isolate, monkeypatch): + _enable(monkeypatch) + _write_cache({"npm install": _cluster()}) + out = proactive.command_cluster_block(_bash("npm install"), now=NOW) + assert isinstance(out, str) + for banned in ("permissionDecision", "deny", "ask", "hookSpecificOutput"): + assert banned not in out # advisory text only, never a gate decision + + def test_token_budget_clip(self, isolate, monkeypatch): + _enable(monkeypatch) + _write_cache({"npm install": _cluster(command="npm install " + "x" * 5000)}) + out = proactive.command_cluster_block(_bash("npm install"), now=NOW) + assert 0 < len(out) <= proactive._CMD_MAX_CHARS + + def test_garbage_payload_is_silent(self, isolate, monkeypatch): + _enable(monkeypatch) + for bad in (None, "x", 42, {}, {"tool_name": "Bash", "tool_input": "nope"}): + assert proactive.command_cluster_block(bad, now=NOW) == "" + + +# ── (7) recall.py wiring — Phase 0 governance retrofit ─────────────────────── + + +def _finding(path="/repo/cost.py", failed=2, reverted=1): + return [{"path": path, "failed": failed, "reverted": reverted, "total": 6, "failure_modes": []}] + + +def _edit(path="/repo/cost.py", session="s1"): + return {"hook_event_name": "PreToolUse", "tool_name": "Edit", + "tool_input": {"file_path": path}, "cwd": "/repo/demo", "session_id": session} + + +class TestRecallGovernance: + def test_passthrough_unchanged_when_disabled(self, isolate, monkeypatch): + # Disabled (default) → recall fires ungoverned, every time, no state file. + monkeypatch.setattr(recall, "_collect_recalls", lambda payload: _finding()) + out1 = recall.build_recall(RECALL_ID, _edit()) + out2 = recall.build_recall(RECALL_ID, _edit()) + assert "cost.py" in out1 and "cost.py" in out2 + assert not proactive._state_path().exists() + + def test_governed_mode_dedupes_file_risk(self, isolate, monkeypatch): + _enable(monkeypatch) + monkeypatch.setattr(recall, "_collect_recalls", lambda payload: _finding()) + out1 = recall.build_recall(RECALL_ID, _edit()) + out2 = recall.build_recall(RECALL_ID, _edit()) + assert "cost.py" in out1 # parity: failed+reverted ≥ 1 still fires + assert out2 == "" # governance throttles the repeat + + def test_file_risk_parity_emits_only_additional_context(self, isolate, monkeypatch): + _enable(monkeypatch) + monkeypatch.setattr(recall, "_collect_recalls", lambda payload: _finding()) + out = recall.build_recall(RECALL_ID, _edit()) + obj = json.loads(out) + assert set(obj) == {"hookSpecificOutput"} + assert set(obj["hookSpecificOutput"]) == {"hookEventName", "additionalContext"} + assert "permissionDecision" not in out and "deny" not in out + + def test_killswitch_silences_recall(self, isolate, monkeypatch): + _enable(monkeypatch) + monkeypatch.setenv("STACKUNDERFLOW_PROACTIVE_DISABLED", "1") + monkeypatch.setattr(recall, "_collect_recalls", lambda payload: _finding()) + assert recall.build_recall(RECALL_ID, _edit()) == "" + + def test_governed_bash_merges_file_and_command_nudges(self, isolate, monkeypatch): + _enable(monkeypatch) + _write_cache({"npm install": _cluster()}) + monkeypatch.setattr(recall, "_collect_recalls", lambda payload: _finding(path="/repo/pkg.json")) + payload = _bash("npm install", session="s9") + out = recall.build_recall(RECALL_ID, payload) + text = json.loads(out)["hookSpecificOutput"]["additionalContext"] + assert "pkg.json" in text # file-risk block + assert "`npm install`" in text # command-cluster block + + +# ── (8) signal-cache precompute (ingest side) ──────────────────────────────── + + +def _seed_store_with_npm_cluster(tmp_path, *, slug): + from stackunderflow.store import db, schema + + conn = db.connect(tmp_path / "store.db") + schema.apply(conn) + pid = int( + conn.execute( + "INSERT INTO projects (provider, slug, display_name, first_seen, last_modified) " + "VALUES ('claude', ?, ?, 0, 0)", + (slug, slug), + ).lastrowid + ) + recent = datetime.now(UTC) - timedelta(days=2) + for i in (1, 2): + sid = f"npm-s{i}" + sfk = int( + conn.execute( + "INSERT INTO sessions (project_id, session_id, first_ts, last_ts, message_count) " + "VALUES (?, ?, NULL, NULL, 0)", + (pid, sid), + ).lastrowid + ) + ts = (recent + timedelta(minutes=i)).isoformat() + tu = f"tu-{i}" + conn.execute( + "INSERT INTO messages (session_fk, seq, timestamp, role, model, input_tokens, " + " output_tokens, cache_create_tokens, cache_read_tokens, content_text, tools_json, " + " raw_json, is_sidechain, uuid, parent_uuid, speed) " + "VALUES (?, 1, ?, 'assistant', '', 0,0,0,0, '', ?, ?, 0, '', NULL, 'standard')", + ( + sfk, ts, + json.dumps([{"id": tu, "name": "Bash", "input": {"command": "npm install --no-fund"}}]), + json.dumps({ + "type": "assistant", "timestamp": ts, + "message": {"role": "assistant", "content": [ + {"type": "tool_use", "id": tu, "name": "Bash", "input": {"command": "npm install --no-fund"}} + ]}, + }), + ), + ) + ts2 = (recent + timedelta(minutes=i, seconds=30)).isoformat() + conn.execute( + "INSERT INTO messages (session_fk, seq, timestamp, role, model, input_tokens, " + " output_tokens, cache_create_tokens, cache_read_tokens, content_text, tools_json, " + " raw_json, is_sidechain, uuid, parent_uuid, speed) " + "VALUES (?, 2, ?, 'user', '', 0,0,0,0, ?, '[]', ?, 0, '', NULL, 'standard')", + ( + sfk, ts2, "Command timed out after 2m 0.0s", + json.dumps({ + "type": "user", "timestamp": ts2, + "message": {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": tu, "is_error": True, + "content": "Command timed out after 2m 0.0s"} + ]}, + }), + ), + ) + conn.commit() + return conn + + +class TestSignalCachePrecompute: + def test_refresh_builds_cache_and_hook_fires(self, isolate, monkeypatch): + _enable(monkeypatch) + slug = proactive._slug_from_cwd("/repo/demo") + conn = _seed_store_with_npm_cluster(isolate, slug=slug) + try: + proactive.refresh_signal_cache(conn, [slug]) + finally: + conn.close() + + cache = json.loads(proactive._signal_path().read_text()) + clusters = cache["projects"][slug]["command_clusters"] + assert "npm install" in clusters + assert clusters["npm install"]["session_count"] == 2 + assert "file_risk" in cache["projects"][slug] + + # And the hook now fires for a matching command in that project (real clock). + out = proactive.command_cluster_block(_bash("cd /repo && npm install")) + assert "`npm install`" in out + + def test_refresh_is_a_noop_when_disabled(self, isolate, monkeypatch): + # No opt-in → no mine_patterns scan, no cache file written. + slug = proactive._slug_from_cwd("/repo/demo") + conn = _seed_store_with_npm_cluster(isolate, slug=slug) + try: + proactive.refresh_signal_cache(conn, [slug]) + finally: + conn.close() + assert not proactive._signal_path().exists() From 13c5655a2f603dac3343be75a905b69d7b4ec262 Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Fri, 3 Jul 2026 10:18:29 -0400 Subject: [PATCH 2/3] =?UTF-8?q?feat(sync):=20multi-device=20encrypted-back?= =?UTF-8?q?up=20MVP=20(#100)=20=E2=80=94=20age/BYO-bucket,=20aggregates-on?= =?UTF-8?q?ly,=20v028?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/specs/session-schema-v1.md | 5 +- pyproject.toml | 9 + stackunderflow/cli.py | 173 +++++++++ .../migrations/v028_sync_identity_outbox.sql | 47 +++ stackunderflow/store/schema.py | 8 +- stackunderflow/sync/__init__.py | 31 ++ stackunderflow/sync/bucket.py | 146 +++++++ stackunderflow/sync/cipher.py | 49 +++ stackunderflow/sync/keys.py | 160 ++++++++ stackunderflow/sync/runner.py | 357 ++++++++++++++++++ stackunderflow/sync/serialize.py | 211 +++++++++++ .../store/test_migration_v028.py | 137 +++++++ tests/stackunderflow/sync/__init__.py | 0 tests/stackunderflow/sync/conftest.py | 130 +++++++ tests/stackunderflow/sync/test_bucket.py | 75 ++++ tests/stackunderflow/sync/test_cipher.py | 32 ++ tests/stackunderflow/sync/test_cli_sync.py | 153 ++++++++ tests/stackunderflow/sync/test_keys.py | 68 ++++ tests/stackunderflow/sync/test_runner.py | 261 +++++++++++++ tests/stackunderflow/sync/test_serialize.py | 133 +++++++ 20 files changed, 2182 insertions(+), 3 deletions(-) create mode 100644 stackunderflow/store/migrations/v028_sync_identity_outbox.sql create mode 100644 stackunderflow/sync/__init__.py create mode 100644 stackunderflow/sync/bucket.py create mode 100644 stackunderflow/sync/cipher.py create mode 100644 stackunderflow/sync/keys.py create mode 100644 stackunderflow/sync/runner.py create mode 100644 stackunderflow/sync/serialize.py create mode 100644 tests/stackunderflow/store/test_migration_v028.py create mode 100644 tests/stackunderflow/sync/__init__.py create mode 100644 tests/stackunderflow/sync/conftest.py create mode 100644 tests/stackunderflow/sync/test_bucket.py create mode 100644 tests/stackunderflow/sync/test_cipher.py create mode 100644 tests/stackunderflow/sync/test_cli_sync.py create mode 100644 tests/stackunderflow/sync/test_keys.py create mode 100644 tests/stackunderflow/sync/test_runner.py create mode 100644 tests/stackunderflow/sync/test_serialize.py diff --git a/docs/specs/session-schema-v1.md b/docs/specs/session-schema-v1.md index ae133101..8ad2e78b 100644 --- a/docs/specs/session-schema-v1.md +++ b/docs/specs/session-schema-v1.md @@ -1,6 +1,6 @@ # Session Schema v1 — open exchange format for AI coding sessions -**Status:** v1 (pinned to `schema_version = 27`). +**Status:** v1 (pinned to `schema_version = 28`). **Audience:** anyone writing a tool that wants to read from, or write to, the StackUnderflow store without reverse-engineering the SQL. **Scope:** the local SQLite schema at `~/.stackunderflow/store.db`. This document is the source of truth for the on-disk shape; the migrations under `stackunderflow/store/migrations/` (`.sql` DDL and `.py` data migrations) are the reference implementation. @@ -20,7 +20,7 @@ The schema described here is **additive-only**. Any future column requires a new ## Schema version -Pin to `schema_version = 27`. The current migration set is: +Pin to `schema_version = 28`. The current migration set is: | version | file | what it adds | |---|---|---| @@ -50,6 +50,7 @@ Pin to `schema_version = 27`. The current migration set is: | 25 | `v025_command_day_mart.sql` | `command_day_mart` (per-`(day, project_id)` user-command count; windows the Overview "Commands" KPI — ui-perf #25) | | 26 | `v026_reasoning_tokens.sql` | adds `usage_events.reasoning_tokens` (reasoning/"thinking" attribution — an additive-metadata SUBSET of `output_tokens`, never priced; `DEFAULT 0`) | | 27 | `v027_worktree_of.sql` | adds `projects.worktree_of` (nullable parent-project slug for git-worktree fragment projects; NULL = normal project — campaign #8) | +| 28 | `v028_sync_identity_outbox.sql` | `sync_identity` + `sync_outbox` (opt-in multi-device sync — device identity + per-shard push watermark; #100 Phase 1) | Version 15 was reserved during planning and never created — the sequence skips from 14 to 16 by design. The migration runner keys on the leading `vNNN`, so the gap is harmless. diff --git a/pyproject.toml b/pyproject.toml index 30cdfdd9..1ea4129c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,15 @@ analysis = [ "mypy>=1.5.0", "coverage>=7.0.0", ] +# Opt-in multi-device sync (docs/specs/multi-device-sync.md). The core product +# never pulls these — they are needed only for the `sync` CLI commands, which +# are import-guarded and print a one-line install hint when the extra is absent. +# `pyrage` = the age file-encryption binding (client-side encryption); `boto3` = +# the S3-compatible bucket client (bring-your-own bucket / credentials). +sync = [ + "boto3>=1.34", + "pyrage>=1.1", +] [project.scripts] stackunderflow = "stackunderflow.cli:cli" diff --git a/stackunderflow/cli.py b/stackunderflow/cli.py index 0c77b06d..19295386 100644 --- a/stackunderflow/cli.py +++ b/stackunderflow/cli.py @@ -1224,6 +1224,179 @@ def _prune_backups(keep: int) -> None: click.echo(f" Pruned old backup: {old.name}") +# ── sync ────────────────────────────────────────────────────────────────────── +# +# Opt-in, client-side-encrypted, bring-your-own-bucket backup of the analytics +# aggregates (docs/specs/multi-device-sync.md, Phase 1 MVP). Default OFF: with +# no ``sync_identity`` row there is no network, no credentials, and the optional +# ``[sync]`` dependencies need not be installed. Raw transcripts, ``usage_events`` +# and the price book NEVER leave the machine — only the derived marts, re-keyed +# from the local ``project_id`` to the machine-stable ``(provider, slug)``. + +_SYNC_INSTALL_HINT = ( + " This command needs optional dependencies that aren't installed.\n" + " Install them with: pip install 'stackunderflow[sync]'" +) + + +def _sync_missing_deps(*, need_bucket: bool) -> list[str]: + """Return the missing optional ``[sync]`` package names (empty = all present).""" + import importlib.util + missing: list[str] = [] + if importlib.util.find_spec("pyrage") is None: + missing.append("pyrage") + if need_bucket and importlib.util.find_spec("boto3") is None: + missing.append("boto3") + return missing + + +def _print_sync_init_banner(identity, device_uuid: str, bucket_url: str) -> None: + """Loud, unmissable zero-knowledge / key-loss warning shown once at ``sync init``.""" + line = " " + "─" * 64 + click.echo(line) + click.echo(" SYNC ENCRYPTION KEY — READ THIS BEFORE YOU CONTINUE") + click.echo(line) + click.echo( + " This device just generated a private encryption key. Everything\n" + " pushed to your bucket is encrypted with it, and NOTHING can decrypt\n" + " it without this key — not the bucket host, not this project, no one.\n" + ) + click.echo( + " IF YOU LOSE THIS KEY, THE OFF-SITE COPY IS UNRECOVERABLE CIPHERTEXT.\n" + " There is no reset, no recovery, no backdoor. That is the point.\n" + ) + click.echo( + " Save the key below in your password manager NOW. To read this data on\n" + " another device, copy the SAME key there — devices must share one key.\n" + ) + click.echo(" Key (store securely — shown once):") + click.echo(f" {identity.secret}") + click.echo("") + click.echo(f" Fingerprint: {identity.fingerprint}") + click.echo(f" Device: {device_uuid}") + click.echo(f" Bucket: {bucket_url}") + click.echo(f" Key file: {_STATE_DIR / 'sync-identity'} (mode 0600)") + click.echo(line) + + +@cli.group("sync") +def sync_group(): + """Encrypted, bring-your-own-bucket backup of your analytics aggregates (opt-in).""" + + +@sync_group.command("init") +@click.option("--bucket", "bucket_url", required=True, + help="Destination bucket, e.g. s3://my-bucket or s3://my-bucket/prefix") +@click.option("--endpoint", "endpoint_url", default=None, + help="Custom object-store endpoint URL (set it for non-default storage providers)") +@click.option("--force", is_flag=True, default=False, + help="Replace an existing sync key on this device (destroys access to data " + "encrypted under the old key — back it up first)") +def sync_init(bucket_url: str, endpoint_url: str | None, force: bool): + """Generate this device's encryption key and record the bucket destination. + + Prints the freshly generated key ONCE — save it, and copy it to your other + devices. Only the key's fingerprint is stored in the database; the secret + lives in a 0600 file (or the keychain / STACKUNDERFLOW_SYNC_KEY env var). + """ + if _sync_missing_deps(need_bucket=False): + click.echo(_SYNC_INSTALL_HINT) + sys.exit(1) + from stackunderflow.sync import keys, runner + + conn = _open_store() + try: + existing = runner.load_identity(conn) + if existing is not None and not force: + click.echo(" Sync is already configured on this device.") + click.echo(f" device: {existing['device_uuid']}") + click.echo(f" key fingerprint: {existing['key_fingerprint']}") + click.echo(" Re-running will NOT change the key. To replace it, back up the") + click.echo(" current key first, then re-run with --force (this destroys access") + click.echo(" to any data already encrypted under the old key).") + sys.exit(1) + + identity = keys.generate_identity() + keys.store_secret_file(identity.secret, _STATE_DIR) + device_uuid = existing["device_uuid"] if existing is not None else runner.new_device_uuid() + runner.write_identity( + conn, + device_uuid=device_uuid, + key_fingerprint=identity.fingerprint, + bucket_url=bucket_url, + endpoint_url=endpoint_url, + created_at=runner.utcnow_iso(), + ) + _print_sync_init_banner(identity, device_uuid, bucket_url) + finally: + conn.close() + + +@sync_group.command("push") +def sync_push(): + """Encrypt and upload changed aggregate shards to your bucket. + + Idempotent — an unchanged shard is skipped (zero uploads). Exits non-zero on + any failure so it is safe to script. + """ + if _sync_missing_deps(need_bucket=True): + click.echo(_SYNC_INSTALL_HINT) + sys.exit(1) + from stackunderflow.sync import runner + + conn = _open_store() + try: + if not runner.is_enabled(conn): + click.echo(" Sync is not configured. Run: stackunderflow sync init --bucket s3://your-bucket") + sys.exit(1) + try: + result = runner.run_push(conn, state_dir=_STATE_DIR) + except Exception as exc: + click.echo(f" sync push failed: {exc}") + sys.exit(1) + finally: + conn.close() + + if result.uploaded == 0: + click.echo(f" Up to date — {result.skipped} shard(s) unchanged, nothing to upload.") + else: + mb = result.bytes_uploaded / (1 << 20) + click.echo(f" Pushed {result.uploaded} shard(s) ({mb:.2f} MB); {result.skipped} unchanged.") + click.echo(f" Generation {result.generation}. Manifest committed.") + + +@sync_group.command("status") +@click.option("--json", "as_json", is_flag=True, default=False, help="Emit machine-readable JSON") +def sync_status(as_json: bool): + """Show sync configuration and how many shards are pending upload (local only).""" + from stackunderflow.sync import runner + + conn = _open_store() + try: + st = runner.status(conn) + finally: + conn.close() + + if as_json: + click.echo(json.dumps(st.as_dict(), indent=2)) + return + + if not st.enabled: + click.echo(" Sync: off (no key on this device).") + click.echo(" Enable with: stackunderflow sync init --bucket s3://your-bucket") + return + + click.echo(" Sync: on") + click.echo(f" device: {st.device_uuid}") + click.echo(f" key fingerprint: {st.fingerprint}") + click.echo(f" bucket: {st.bucket_url}") + if st.endpoint_url: + click.echo(f" endpoint: {st.endpoint_url}") + click.echo(f" shards (local): {st.shard_count}") + click.echo(f" pending upload: {len(st.pending)}") + click.echo(f" last push: {st.last_push_ts or 'never'}") + + # ── data commands ──────────────────────────────────────────────────────────── _VALID_FORMATS = ("text", "json") diff --git a/stackunderflow/store/migrations/v028_sync_identity_outbox.sql b/stackunderflow/store/migrations/v028_sync_identity_outbox.sql new file mode 100644 index 00000000..12f76c32 --- /dev/null +++ b/stackunderflow/store/migrations/v028_sync_identity_outbox.sql @@ -0,0 +1,47 @@ +-- v028: opt-in multi-device sync — device identity + push outbox (Phase 1 MVP). +-- +-- Foundation tables for ``docs/specs/multi-device-sync.md`` Phase 1 (one-way, +-- client-side-encrypted backup of the analytics aggregates to the user's own +-- bucket). Only two tables ship in the MVP: +-- +-- * ``sync_identity`` — single row (CHECK id = 1): this device's random UUID, +-- the encryption-key FINGERPRINT (never the secret — that lives in the +-- keychain / a 0600 file / an env var), and the destination bucket config. +-- * ``sync_outbox`` — per-shard PUSH watermark: the last content-hash we +-- uploaded for each ``(mart family, month)`` shard, so an unchanged shard is +-- skipped (idempotent push). ``sync_cursors`` / ``sync_remote_devices`` and +-- the ``_remote`` landing tables are Phase 2 (pull/merge) and NOT created here. +-- +-- Migration is **additive** — no existing table is touched, so a store with +-- sync disabled (no ``sync_identity`` row) is byte-for-byte unchanged and every +-- existing query behaves exactly as before. Both CREATEs are ``IF NOT EXISTS`` +-- and the loader's ``_ADD_COLUMN_GUARDS`` entry ``("sync_identity", "device_uuid")`` +-- makes a partial prior run (table present, ``user_version`` behind) bump the +-- version without re-executing the body. + +BEGIN; + +-- Device identity + bucket config. Single row (id = 1). +CREATE TABLE IF NOT EXISTS sync_identity ( + id INTEGER PRIMARY KEY CHECK (id = 1), + device_uuid TEXT NOT NULL, -- random, minted at `sync init` + key_fingerprint TEXT NOT NULL, -- fingerprint ONLY; secret never here + bucket_url TEXT NOT NULL, + endpoint_url TEXT, -- NULL = AWS default; set for R2/B2/MinIO + layout_version INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL +); + +-- Push watermark: what this device owes the bucket, keyed by logical shard. +CREATE TABLE IF NOT EXISTS sync_outbox ( + shard_key TEXT PRIMARY KEY, -- "daily_mart.2026-07" + content_hash TEXT, -- current local plaintext hash + generation INTEGER NOT NULL DEFAULT 0, + dirty INTEGER NOT NULL DEFAULT 1, + last_pushed_hash TEXT, + last_pushed_ts TEXT +); + +PRAGMA user_version = 28; + +COMMIT; diff --git a/stackunderflow/store/schema.py b/stackunderflow/store/schema.py index 32d27c3a..129380e0 100644 --- a/stackunderflow/store/schema.py +++ b/stackunderflow/store/schema.py @@ -26,7 +26,7 @@ _MIGRATIONS_DIR = Path(__file__).parent / "migrations" -CURRENT_VERSION = 27 +CURRENT_VERSION = 28 def apply(conn: sqlite3.Connection) -> None: @@ -117,6 +117,12 @@ def _run_python_migration( # guard so a partial prior run (column added, ``user_version`` behind) # bumps the version instead of erroring on "duplicate column". 27: ("projects", "worktree_of"), + # v028 CREATEs ``sync_identity`` + ``sync_outbox`` (opt-in multi-device sync, + # Phase 1). Both are ``CREATE TABLE IF NOT EXISTS`` so re-running is already + # safe; the guard makes the partial-application path — table present, + # ``user_version`` behind — bump the version without re-executing the body. + # ``_column_exists`` doubles as a "does this table exist with this column?" probe. + 28: ("sync_identity", "device_uuid"), } diff --git a/stackunderflow/sync/__init__.py b/stackunderflow/sync/__init__.py new file mode 100644 index 00000000..db6e16d9 --- /dev/null +++ b/stackunderflow/sync/__init__.py @@ -0,0 +1,31 @@ +"""Opt-in, client-side-encrypted, bring-your-own-bucket sync (Phase 1 MVP). + +Implements ``docs/specs/multi-device-sync.md`` Phase 1: a one-way, encrypted +backup of the *analytics aggregates* (the Overview/Cost-core marts) to the +user's own S3-compatible bucket. Zero-knowledge — the bucket stores ciphertext +only. Raw transcripts, ``usage_events`` and the ``price_book`` NEVER leave the +machine; only the derived marts move, re-keyed from the machine-local +``project_id`` to the stable ``(provider, slug)`` identity. + +**Default OFF.** With no ``sync_identity`` row there is no network, no +credentials, and the optional ``[sync]`` dependencies (``pyrage`` + ``boto3``) +need not be installed. Every module here keeps its optional-dependency imports +*inside functions* so importing the package never fails on a core install. + +Layering: + +* :mod:`stackunderflow.sync.keys` — age identity resolution (env → keychain → + ``0600`` file) and fingerprints. Only the fingerprint is ever stored in the DB. +* :mod:`stackunderflow.sync.cipher` — ``age`` encrypt/decrypt via ``pyrage``. +* :mod:`stackunderflow.sync.bucket` — the narrow ``ObjectStore`` interface, a + ``boto3`` implementation and an in-memory fake for tests. +* :mod:`stackunderflow.sync.serialize` — canonical, deterministic mart-shard + serialization + SHA-256 content-hash, and the ``project_id`` → ``(provider, + slug)`` re-keying. +* :mod:`stackunderflow.sync.runner` — ``init`` / ``push`` / ``status`` with a + two-phase manifest commit and a skip-if-unchanged outbox. +""" + +from __future__ import annotations + +__all__ = ["bucket", "cipher", "keys", "runner", "serialize"] diff --git a/stackunderflow/sync/bucket.py b/stackunderflow/sync/bucket.py new file mode 100644 index 00000000..7239fd0e --- /dev/null +++ b/stackunderflow/sync/bucket.py @@ -0,0 +1,146 @@ +"""The narrow ``ObjectStore`` interface + a boto3 impl and an in-memory fake. + +The sync logic depends only on ``put/get/list/delete`` so S3 quirks stay out of +the protocol and the test suite is hermetic (it runs against +:class:`InMemoryObjectStore`, never a real bucket). ``boto3`` is imported on +demand — a core install never pulls it. +""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from .keys import SyncDependencyError + + +class ObjectNotFound(KeyError): + """Raised by :meth:`ObjectStore.get` when *key* is absent.""" + + +@runtime_checkable +class ObjectStore(Protocol): + """A minimal object store: bytes in, bytes out, list by prefix, delete.""" + + def put(self, key: str, data: bytes) -> None: ... + + def get(self, key: str) -> bytes: ... + + def list(self, prefix: str) -> list[str]: ... # noqa: A003 - part of the public interface name + + def delete(self, key: str) -> None: ... + + +class InMemoryObjectStore: + """A dict-backed :class:`ObjectStore` for tests. Counts calls for assertions.""" + + def __init__(self) -> None: + self._data: dict[str, bytes] = {} + self.put_calls = 0 + self.get_calls = 0 + self.list_calls = 0 + self.delete_calls = 0 + + def put(self, key: str, data: bytes) -> None: + self.put_calls += 1 + self._data[key] = bytes(data) + + def get(self, key: str) -> bytes: + self.get_calls += 1 + try: + return self._data[key] + except KeyError as exc: + raise ObjectNotFound(key) from exc + + def list(self, prefix: str) -> list[str]: # noqa: A003 - part of the public interface name + self.list_calls += 1 + return sorted(k for k in self._data if k.startswith(prefix)) + + def delete(self, key: str) -> None: + self.delete_calls += 1 + self._data.pop(key, None) + + # Test convenience — not part of the ObjectStore protocol. + def __contains__(self, key: str) -> bool: + return key in self._data + + def __len__(self) -> int: + return len(self._data) + + +def parse_bucket_url(url: str) -> tuple[str, str]: + """Split an ``s3://bucket[/prefix]`` URL into ``(bucket, key_prefix)``. + + *key_prefix* is ``""`` when the URL names only a bucket. + """ + if not url.startswith("s3://"): + raise ValueError(f"bucket URL must start with s3:// — got {url!r}") + rest = url[len("s3://"):] + parts = rest.split("/", 1) + bucket = parts[0] + if not bucket: + raise ValueError(f"bucket URL has no bucket name — got {url!r}") + prefix = parts[1].strip("/") if len(parts) > 1 else "" + return bucket, prefix + + +class S3ObjectStore: + """A ``boto3``-backed :class:`ObjectStore` (AWS S3, R2, B2, MinIO, …). + + Credentials come from the user's own standard AWS chain (``~/.aws``, + ``AWS_*`` env) or the boto3 *client* passed in — never a StackUnderflow-issued + key, and never persisted to ``store.db`` / ``config.json``. + """ + + def __init__( + self, + bucket: str, + *, + key_prefix: str = "", + endpoint_url: str | None = None, + client=None, + ) -> None: + self._bucket = bucket + self._prefix = key_prefix.strip("/") + if client is not None: + self._client = client + else: + try: + import boto3 + except ImportError as exc: # pragma: no cover - exercised via CLI hint path + raise SyncDependencyError( + "the 'boto3' package is required to talk to a bucket; " + "install with: pip install 'stackunderflow[sync]'" + ) from exc + self._client = boto3.client("s3", endpoint_url=endpoint_url) + + def _full(self, key: str) -> str: + return f"{self._prefix}/{key}" if self._prefix else key + + def put(self, key: str, data: bytes) -> None: + self._client.put_object(Bucket=self._bucket, Key=self._full(key), Body=data) + + def get(self, key: str) -> bytes: + try: + resp = self._client.get_object(Bucket=self._bucket, Key=self._full(key)) + except Exception as exc: # botocore ClientError (NoSuchKey) et al. + raise ObjectNotFound(key) from exc + return resp["Body"].read() + + def list(self, prefix: str) -> list[str]: # noqa: A003 - part of the public interface name + full_prefix = self._full(prefix) + strip = len(self._prefix) + 1 if self._prefix else 0 + keys: list[str] = [] + paginator = self._client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=self._bucket, Prefix=full_prefix): + for obj in page.get("Contents", []): + keys.append(obj["Key"][strip:] if strip else obj["Key"]) + return sorted(keys) + + def delete(self, key: str) -> None: + self._client.delete_object(Bucket=self._bucket, Key=self._full(key)) + + +def s3_store_from_url(bucket_url: str, endpoint_url: str | None = None) -> S3ObjectStore: + """Build an :class:`S3ObjectStore` from an ``s3://…`` URL (imports boto3).""" + bucket, prefix = parse_bucket_url(bucket_url) + return S3ObjectStore(bucket, key_prefix=prefix, endpoint_url=endpoint_url) diff --git a/stackunderflow/sync/cipher.py b/stackunderflow/sync/cipher.py new file mode 100644 index 00000000..61454209 --- /dev/null +++ b/stackunderflow/sync/cipher.py @@ -0,0 +1,49 @@ +"""``age`` encryption for sync shards — no rolled-own crypto. + +Every blob is encrypted with ``age`` (an audited file-encryption format) via the +``pyrage`` binding: ephemeral X25519 → HKDF-SHA256 → ChaCha20-Poly1305 over a +chunked STREAM construction. We only ever call ``encrypt(recipient, plaintext)`` +/ ``decrypt(identity, bytes)``; nothing lower-level (nonces, AEAD, key schedule) +is our responsibility. + +``pyrage`` is imported on demand so a core install (no ``[sync]`` extra) can +still import this module. +""" + +from __future__ import annotations + +from .keys import SyncDependencyError, _pyrage + + +class DecryptError(RuntimeError): + """A blob could not be decrypted (wrong key, or corrupt/tampered ciphertext). + + ``age``'s per-frame AEAD authenticates every shard, so a truncated, swapped + or tampered blob fails to decrypt rather than returning a silent partial + read. A wrong key raises this too — cleanly, with no local mutation. + """ + + +def encrypt(plaintext: bytes, recipient: str) -> bytes: + """Encrypt *plaintext* to the age *recipient* (``age1…``). Returns ciphertext.""" + pyrage = _pyrage() + recip = pyrage.x25519.Recipient.from_str(recipient.strip()) + return pyrage.encrypt(plaintext, [recip]) + + +def decrypt(ciphertext: bytes, secret: str) -> bytes: + """Decrypt *ciphertext* with the age *secret* (``AGE-SECRET-KEY-1…``). + + Raises :class:`DecryptError` on a wrong key or a corrupt/tampered blob. + """ + pyrage = _pyrage() + ident = pyrage.x25519.Identity.from_str(secret.strip()) + try: + return pyrage.decrypt(ciphertext, [ident]) + except SyncDependencyError: + raise + except Exception as exc: + raise DecryptError( + "could not decrypt — the blob is not encrypted for your key, " + "or it is corrupt/tampered" + ) from exc diff --git a/stackunderflow/sync/keys.py b/stackunderflow/sync/keys.py new file mode 100644 index 00000000..9906c17a --- /dev/null +++ b/stackunderflow/sync/keys.py @@ -0,0 +1,160 @@ +"""Age identity management for sync (Phase 1 MVP). + +The sync key is an ``age`` X25519 identity (``AGE-SECRET-KEY-1…``). It is the +user's secret; losing it makes the off-site ciphertext unrecoverable — that is +what zero-knowledge means. The key therefore never sits in ``store.db`` or +``config.json``; only its *fingerprint* is persisted (in ``sync_identity``). + +Resolution order on read mirrors the ``_Opt`` secret-shaped chain in +``settings.py``: env ``STACKUNDERFLOW_SYNC_KEY`` → OS keychain → ``0600`` +file at ``/sync-identity``. + +Only :func:`generate_identity` and :func:`recipient_for` need ``pyrage``; the +resolution / storage / fingerprint helpers are dependency-free so the file and +env legs work (and are testable) on a core install. +""" + +from __future__ import annotations + +import hashlib +import os +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +#: Environment variable that, when set, is the highest-priority key source. +ENV_KEY = "STACKUNDERFLOW_SYNC_KEY" + +#: macOS keychain service name for a manually-stored key (read-only leg). +KEYCHAIN_SERVICE = "stackunderflow-sync" + +#: Filename of the ``0600`` on-disk key inside the state dir. +IDENTITY_FILENAME = "sync-identity" + + +class SyncDependencyError(RuntimeError): + """Raised when an optional ``[sync]`` dependency is not installed.""" + + +@dataclass(frozen=True) +class AgeIdentity: + """An age identity: the secret, its public recipient, and a fingerprint.""" + + secret: str + recipient: str + fingerprint: str + + +def _pyrage(): + try: + import pyrage + except ImportError as exc: # pragma: no cover - exercised via CLI hint path + raise SyncDependencyError( + "the 'pyrage' package is required for sync crypto; " + "install with: pip install 'stackunderflow[sync]'" + ) from exc + return pyrage + + +def generate_identity() -> AgeIdentity: + """Generate a fresh random X25519 age identity.""" + pyrage = _pyrage() + ident = pyrage.x25519.Identity.generate() + secret = str(ident) + recipient = str(ident.to_public()) + return AgeIdentity(secret=secret, recipient=recipient, fingerprint=fingerprint(recipient)) + + +def recipient_for(secret: str) -> str: + """Return the public recipient (``age1…``) for a secret identity string.""" + pyrage = _pyrage() + ident = pyrage.x25519.Identity.from_str(secret.strip()) + return str(ident.to_public()) + + +def fingerprint(recipient: str) -> str: + """Short, stable fingerprint of a recipient — for display and key-mismatch checks. + + A truncated SHA-256 of the recipient string. Dependency-free (no ``pyrage``) + so a key-mismatch check can run without the crypto extra installed. + """ + return hashlib.sha256(recipient.encode("utf-8")).hexdigest()[:16] + + +def identity_path(state_dir: os.PathLike[str] | str) -> Path: + """Path of the on-disk ``0600`` key inside *state_dir*.""" + return Path(state_dir) / IDENTITY_FILENAME + + +def _read_keychain(service: str = KEYCHAIN_SERVICE) -> str | None: + """Best-effort, READ-ONLY macOS keychain lookup. Never writes, never raises. + + Returns the stored secret if the user manually added a generic-password item + under *service*, else ``None``. On non-macOS, or on any error, returns + ``None``. We never *write* to the keychain — that would mutate system state; + the on-disk ``0600`` file is the storage default (see :func:`store_secret_file`). + """ + if sys.platform != "darwin": + return None + try: + import subprocess + + result = subprocess.run( # noqa: S603, S607 - fixed argv, no shell, no user input + ["security", "find-generic-password", "-w", "-s", service], + capture_output=True, + text=True, + timeout=5, + ) + except Exception: # pragma: no cover - defensive; keychain is a best-effort leg + return None + if result.returncode == 0: + value = result.stdout.strip() + return value or None + return None + + +def resolve_secret( + state_dir: os.PathLike[str] | str, + *, + env: dict[str, str] | None = None, + keychain_reader: Callable[[], str | None] | None = None, +) -> str | None: + """Resolve the sync secret: env → keychain → ``0600`` file. ``None`` if unset. + + *env* defaults to ``os.environ``; *keychain_reader* defaults to the + read-only macOS lookup — both are injectable so tests stay hermetic (they + never shell out to ``security``). + """ + environ = os.environ if env is None else env + from_env = environ.get(ENV_KEY) + if from_env and from_env.strip(): + return from_env.strip() + + reader = keychain_reader if keychain_reader is not None else _read_keychain + from_keychain = reader() + if from_keychain and from_keychain.strip(): + return from_keychain.strip() + + path = identity_path(state_dir) + if path.exists(): + text = path.read_text().strip() + return text or None + return None + + +def store_secret_file(secret: str, state_dir: os.PathLike[str] | str) -> Path: + """Write *secret* to ``/sync-identity`` with mode ``0600``. + + This is the storage default at ``sync init``. It touches only the project + state dir — never the keychain (a system-state mutation) — so enabling sync + has no side effects outside ``~/.stackunderflow``. + """ + path = identity_path(state_dir) + path.parent.mkdir(parents=True, exist_ok=True) + # Open with 0600 up front (subject to umask), then chmod to be certain. + fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as handle: + handle.write(secret.strip() + "\n") + os.chmod(path, 0o600) + return path diff --git a/stackunderflow/sync/runner.py b/stackunderflow/sync/runner.py new file mode 100644 index 00000000..347e95cb --- /dev/null +++ b/stackunderflow/sync/runner.py @@ -0,0 +1,357 @@ +"""``sync init`` / ``push`` / ``status`` orchestration (Phase 1 MVP). + +The core :func:`push` is dependency-free: crypto and the object store are +*injected* (an ``encryptor`` callable and an :class:`~stackunderflow.sync.bucket.ObjectStore`), +so idempotency / outbox / two-phase-commit behaviour is fully testable without +``pyrage`` or ``boto3``. :func:`run_push` is the thin deps-wiring wrapper the CLI +uses: it resolves the key (``pyrage``), binds the age cipher, and builds the S3 +store (``boto3``). + +Push is two-phase and crash-safe (§4.2): + +1. Upload every changed shard object (``PUT`` is atomic per object). +2. Overwrite ``manifest.age`` last — the only object a puller trusts. + +A crash between phases leaves orphan shards the current manifest doesn't +reference; a reader never sees a half-applied state. Idempotency (§5.4): a shard +whose content-hash equals its ``last_pushed_hash`` is skipped, and when nothing +changed the manifest is not rewritten either — zero puts. +""" + +from __future__ import annotations + +import json +import sqlite3 +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Callable + +from . import serialize + +#: Object-key layout root (readable keys — the MVP default, §4.1/§11). +DEFAULT_PREFIX = "stackunderflow/v1" + +#: Manifest schema tag embedded in the (encrypted) manifest. +MANIFEST_SCHEMA = "stackunderflow.sync/1" + +Encryptor = Callable[[bytes], bytes] + + +class SyncError(RuntimeError): + """Base class for sync operational errors.""" + + +class SyncNotConfigured(SyncError): + """No ``sync_identity`` row — ``sync init`` has not been run on this device.""" + + +class SyncKeyMissing(SyncError): + """The sync key could not be resolved (env / keychain / ``0600`` file).""" + + +class SyncKeyMismatch(SyncError): + """The resolved key does not match the fingerprint recorded at ``sync init``.""" + + +def utcnow_iso() -> str: + """Wall-clock UTC timestamp (seconds precision). Not part of any content hash.""" + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def new_device_uuid() -> str: + """A random device UUID, not tied to hostname or user (§4.1).""" + return uuid.uuid4().hex + + +# ── object keys ─────────────────────────────────────────────────────────────── + + +def object_key(device_uuid: str, shard_key: str, *, prefix: str = DEFAULT_PREFIX) -> str: + """Readable per-device shard object key (§4.1).""" + return f"{prefix}/{device_uuid}/shards/{shard_key}.age" + + +def manifest_key(device_uuid: str, *, prefix: str = DEFAULT_PREFIX) -> str: + """Per-device manifest object key (the commit point).""" + return f"{prefix}/{device_uuid}/manifest.age" + + +# ── identity (sync_identity table) ──────────────────────────────────────────── + + +def load_identity(conn: sqlite3.Connection) -> dict | None: + """Return the single ``sync_identity`` row as a dict, or ``None`` if unset.""" + row = conn.execute( + "SELECT device_uuid, key_fingerprint, bucket_url, endpoint_url, " + " layout_version, created_at " + "FROM sync_identity WHERE id = 1" + ).fetchone() + return dict(row) if row is not None else None + + +def is_enabled(conn: sqlite3.Connection) -> bool: + """True when this device has a ``sync_identity`` row (sync is opted in).""" + return load_identity(conn) is not None + + +def write_identity( + conn: sqlite3.Connection, + *, + device_uuid: str, + key_fingerprint: str, + bucket_url: str, + endpoint_url: str | None, + created_at: str, + layout_version: int = 1, +) -> None: + """Insert (or replace) the single-row ``sync_identity`` record.""" + conn.execute( + "INSERT OR REPLACE INTO sync_identity " + "(id, device_uuid, key_fingerprint, bucket_url, endpoint_url, layout_version, created_at) " + "VALUES (1, ?, ?, ?, ?, ?, ?)", + (device_uuid, key_fingerprint, bucket_url, endpoint_url, layout_version, created_at), + ) + + +# ── outbox (sync_outbox table) ──────────────────────────────────────────────── + + +def _load_outbox(conn: sqlite3.Connection) -> dict[str, dict]: + rows = conn.execute( + "SELECT shard_key, content_hash, generation, dirty, last_pushed_hash, last_pushed_ts " + "FROM sync_outbox" + ).fetchall() + return {r["shard_key"]: dict(r) for r in rows} + + +def _record_pushed( + conn: sqlite3.Connection, + shard_key: str, + *, + content_hash: str, + generation: int, + pushed_at: str, +) -> None: + conn.execute( + "INSERT INTO sync_outbox " + "(shard_key, content_hash, generation, dirty, last_pushed_hash, last_pushed_ts) " + "VALUES (?, ?, ?, 0, ?, ?) " + "ON CONFLICT(shard_key) DO UPDATE SET " + " content_hash = excluded.content_hash, " + " generation = excluded.generation, " + " dirty = 0, " + " last_pushed_hash = excluded.last_pushed_hash, " + " last_pushed_ts = excluded.last_pushed_ts", + (shard_key, content_hash, generation, content_hash, pushed_at), + ) + + +# ── push ────────────────────────────────────────────────────────────────────── + + +@dataclass +class PushResult: + """Outcome of a :func:`push`.""" + + uploaded: int + skipped: int + bytes_uploaded: int + generation: int + manifest_written: bool + shard_keys: list[str] = field(default_factory=list) + + +def push( + conn: sqlite3.Connection, + store, + *, + device_uuid: str, + key_fingerprint: str, + encryptor: Encryptor, + now: str | None = None, + prefix: str = DEFAULT_PREFIX, +) -> PushResult: + """Encrypt and upload changed aggregate shards, then commit the manifest. + + Pure w.r.t. optional dependencies — *encryptor* and *store* are injected. + Idempotent: unchanged shards are skipped and, when nothing changed, the + manifest is not rewritten (zero puts). Raises whatever *store.put* raises; + on a mid-push failure the already-recorded outbox rows persist (autocommit) + while the failed shard stays un-pushed and the manifest is not written, so a + retry re-uploads and readers keep the previous manifest. + """ + now = now or utcnow_iso() + shards = serialize.build_shards(conn) + outbox = _load_outbox(conn) + current_gen = max((row["generation"] for row in outbox.values()), default=0) + + manifest_shards: dict[str, dict] = {} + to_upload: list[tuple[str, str, bytes, str]] = [] # (shard_key, object_key, body, hash) + for shard in shards: + body = shard.to_bytes() + content_hash = shard.content_hash + key = object_key(device_uuid, shard.shard_key, prefix=prefix) + manifest_shards[shard.shard_key] = { + "object_key": key, + "content_hash": content_hash, + "bytes": len(body), + } + prev = outbox.get(shard.shard_key) + if prev is None or prev["last_pushed_hash"] != content_hash: + to_upload.append((shard.shard_key, key, body, content_hash)) + + if not to_upload: + # Fully idempotent no-op: nothing changed ⇒ no puts, no manifest rewrite. + return PushResult( + uploaded=0, + skipped=len(shards), + bytes_uploaded=0, + generation=current_gen, + manifest_written=False, + ) + + new_gen = current_gen + 1 + total_bytes = 0 + # Phase 1 — upload changed shard objects (each PUT is atomic per object). + for shard_key, key, body, content_hash in to_upload: + ciphertext = encryptor(body) + store.put(key, ciphertext) + total_bytes += len(ciphertext) + _record_pushed(conn, shard_key, content_hash=content_hash, generation=new_gen, pushed_at=now) + + # Phase 2 — overwrite the manifest last (the only object a puller trusts). + manifest = { + "schema": MANIFEST_SCHEMA, + "device_uuid": device_uuid, + "key_fingerprint": key_fingerprint, + "generation": new_gen, + "created_at": now, + "layout_version": 1, + "shards": manifest_shards, + } + manifest_bytes = json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8") + store.put(manifest_key(device_uuid, prefix=prefix), encryptor(manifest_bytes)) + + return PushResult( + uploaded=len(to_upload), + skipped=len(shards) - len(to_upload), + bytes_uploaded=total_bytes, + generation=new_gen, + manifest_written=True, + shard_keys=[sk for sk, _, _, _ in to_upload], + ) + + +def run_push( + conn: sqlite3.Connection, + *, + state_dir, + store=None, + env: dict[str, str] | None = None, + now: str | None = None, +) -> PushResult: + """Resolve the key + bucket, then :func:`push`. Wires the optional deps. + + Raises :class:`SyncNotConfigured` / :class:`SyncKeyMissing` / + :class:`SyncKeyMismatch` for the config/key failure modes. + """ + from . import bucket, cipher, keys + + identity = load_identity(conn) + if identity is None: + raise SyncNotConfigured("sync is not configured — run `stackunderflow sync init` first") + + secret = keys.resolve_secret(state_dir, env=env) + if secret is None: + raise SyncKeyMissing( + "no sync key found — set STACKUNDERFLOW_SYNC_KEY, add it to the keychain, " + f"or place it at {keys.identity_path(state_dir)}" + ) + + recipient = keys.recipient_for(secret) + if keys.fingerprint(recipient) != identity["key_fingerprint"]: + raise SyncKeyMismatch( + "the resolved key does not match the fingerprint recorded at `sync init` " + f"({identity['key_fingerprint']}) — check STACKUNDERFLOW_SYNC_KEY / the key file" + ) + + def _encrypt(plaintext: bytes) -> bytes: + return cipher.encrypt(plaintext, recipient) + + if store is None: + store = bucket.s3_store_from_url(identity["bucket_url"], identity["endpoint_url"]) + + return push( + conn, + store, + device_uuid=identity["device_uuid"], + key_fingerprint=identity["key_fingerprint"], + encryptor=_encrypt, + now=now, + ) + + +# ── status ──────────────────────────────────────────────────────────────────── + + +@dataclass +class SyncStatus: + """Local sync state — computed without any network or optional dependency.""" + + enabled: bool + device_uuid: str | None = None + fingerprint: str | None = None + bucket_url: str | None = None + endpoint_url: str | None = None + shard_count: int = 0 + pending: list[str] = field(default_factory=list) + last_push_ts: str | None = None + + def as_dict(self) -> dict: + return { + "enabled": self.enabled, + "device_uuid": self.device_uuid, + "fingerprint": self.fingerprint, + "bucket_url": self.bucket_url, + "endpoint_url": self.endpoint_url, + "shard_count": self.shard_count, + "pending": list(self.pending), + "pending_count": len(self.pending), + "last_push_ts": self.last_push_ts, + } + + +def status(conn: sqlite3.Connection) -> SyncStatus: + """Report sync config + how many local shards are pending upload. + + Purely local: reads ``sync_identity`` + ``sync_outbox`` and rebuilds the + shards to diff their content-hashes against ``last_pushed_hash``. No network, + no crypto, no optional dependency — safe on a core install. + """ + identity = load_identity(conn) + if identity is None: + return SyncStatus(enabled=False) + + shards = serialize.build_shards(conn) + outbox = _load_outbox(conn) + pending = [ + shard.shard_key + for shard in shards + if (prev := outbox.get(shard.shard_key)) is None + or prev["last_pushed_hash"] != shard.content_hash + ] + last_push = max( + (r["last_pushed_ts"] for r in outbox.values() if r["last_pushed_ts"]), + default=None, + ) + return SyncStatus( + enabled=True, + device_uuid=identity["device_uuid"], + fingerprint=identity["key_fingerprint"], + bucket_url=identity["bucket_url"], + endpoint_url=identity["endpoint_url"], + shard_count=len(shards), + pending=pending, + last_push_ts=last_push, + ) diff --git a/stackunderflow/sync/serialize.py b/stackunderflow/sync/serialize.py new file mode 100644 index 00000000..fe36b10e --- /dev/null +++ b/stackunderflow/sync/serialize.py @@ -0,0 +1,211 @@ +"""Canonical, deterministic mart-shard serialization + re-keying. + +A **shard** is one ``(mart family, month)`` — e.g. ``daily_mart`` for +``2026-07``. Each shard serializes to a canonical, deterministic byte form +(sorted rows, fixed field order, no wall-clock / random) so a given dataset +always hashes identically. The SHA-256 of those bytes is the shard's +**content-hash**, which drives push idempotency (§5.4 of the spec). + +**Re-keying (§4.5).** ``projects.id`` is a machine-local autoincrement, so raw +mart rows are not cross-device-comparable. At export every shard is re-keyed +from the local ``project_id`` to the machine-stable identity ``(provider, +slug)`` via a ``JOIN projects`` at serialize time, and grouped/summed at that +stable grain. Two machines that assign different local ids to the same +``(provider, slug)`` therefore produce identical shard bytes — the property the +cross-device union relies on. + +**What ships in the MVP** — the Overview/Cost core: ``daily_mart``, +``project_mart``, ``provider_day_mart``, ``model_day_mart``, ``session_mart``. +``message_tool_mart`` (carries ``file_path``) is excluded. ``usage_events`` and +``price_book`` are never read here — only the derived marts move, and +``session_mart.cwd`` (a filesystem path) is deliberately dropped. +""" + +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from collections import defaultdict +from dataclasses import dataclass + + +@dataclass(frozen=True) +class _MartSpec: + """A mart family's export query, canonical columns, and month grouping.""" + + family: str + columns: tuple[str, ...] + sql: str + # Column whose value's ``YYYY-MM`` prefix buckets rows into monthly shards. + # ``None`` means the whole mart is a single ``"all"`` shard. + month_column: str | None + + +# The five Overview/Cost-core marts, each re-keyed to ``(provider, slug)`` where +# it carries a local ``project_id``. ``provider_day_mart`` / ``model_day_mart`` +# carry no ``project_id`` and need no re-key. ``session_mart.cwd`` is dropped +# (path-shaped). SELECT column order MUST match ``columns``. +_SPECS: tuple[_MartSpec, ...] = ( + _MartSpec( + family="daily_mart", + columns=( + "day", "provider", "slug", "model", "speed", + "input_tokens", "output_tokens", "cache_read", "cache_create", + "message_count", "session_count", "cost_usd", + ), + sql=( + "SELECT d.day, d.provider, p.slug, d.model, d.speed, " + " SUM(d.input_tokens), SUM(d.output_tokens), " + " SUM(d.cache_read), SUM(d.cache_create), " + " SUM(d.message_count), SUM(d.session_count), SUM(d.cost_usd) " + "FROM daily_mart d JOIN projects p ON p.id = d.project_id " + "GROUP BY d.day, d.provider, p.slug, d.model, d.speed " + "ORDER BY d.day, d.provider, p.slug, d.model, d.speed" + ), + month_column="day", + ), + _MartSpec( + family="provider_day_mart", + columns=( + "day", "provider", "cost_usd", + "message_count", "session_count", "project_count", + ), + sql=( + "SELECT day, provider, cost_usd, message_count, session_count, project_count " + "FROM provider_day_mart " + "ORDER BY day, provider" + ), + month_column="day", + ), + _MartSpec( + family="model_day_mart", + columns=( + "day", "model", "speed", "cost_usd", + "input_tokens", "output_tokens", "cache_read", "cache_create", + "message_count", "session_count", + ), + sql=( + "SELECT day, model, speed, cost_usd, input_tokens, output_tokens, " + " cache_read, cache_create, message_count, session_count " + "FROM model_day_mart " + "ORDER BY day, model, speed" + ), + month_column="day", + ), + _MartSpec( + family="project_mart", + columns=( + "provider", "slug", "display_name", "first_ts", "last_ts", + "total_messages", "total_sessions", "total_input_tokens", + "total_output_tokens", "total_cache_read", "total_cache_create", + "total_cost_usd", + ), + sql=( + "SELECT provider, slug, display_name, first_ts, last_ts, " + " SUM(total_messages), SUM(total_sessions), SUM(total_input_tokens), " + " SUM(total_output_tokens), SUM(total_cache_read), " + " SUM(total_cache_create), SUM(total_cost_usd) " + "FROM project_mart " + "GROUP BY provider, slug, display_name, first_ts, last_ts " + "ORDER BY provider, slug" + ), + month_column=None, + ), + _MartSpec( + family="session_mart", + columns=( + "session_id", "provider", "slug", "primary_model", "first_ts", "last_ts", + "message_count", "user_message_count", "assistant_message_count", + "input_tokens", "output_tokens", "cache_read", "cache_create", + "cost_usd", "is_one_shot", + ), + sql=( + "SELECT s.session_id, s.provider, p.slug, s.primary_model, " + " s.first_ts, s.last_ts, s.message_count, s.user_message_count, " + " s.assistant_message_count, s.input_tokens, s.output_tokens, " + " s.cache_read, s.cache_create, s.cost_usd, s.is_one_shot " + "FROM session_mart s JOIN projects p ON p.id = s.project_id " + "ORDER BY s.session_id" + ), + month_column="first_ts", + ), +) + +#: The mart families the MVP syncs, in a stable order. +MART_FAMILIES: tuple[str, ...] = tuple(spec.family for spec in _SPECS) + +#: Serialization format version, embedded in each shard's canonical bytes. +FORMAT_VERSION = 1 + + +@dataclass(frozen=True) +class Shard: + """One ``(family, month)`` shard of re-keyed, canonically-ordered mart rows.""" + + family: str + month: str # "YYYY-MM", or "all" for month-less marts (project_mart) + columns: tuple[str, ...] + rows: tuple[tuple, ...] + + @property + def shard_key(self) -> str: + """Stable logical key, e.g. ``daily_mart.2026-07`` / ``project_mart.all``.""" + return f"{self.family}.{self.month}" + + def to_bytes(self) -> bytes: + """Canonical, deterministic serialization (drives the content hash).""" + payload = { + "v": FORMAT_VERSION, + "family": self.family, + "month": self.month, + "columns": list(self.columns), + "rows": [list(row) for row in self.rows], + } + return json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + + @property + def content_hash(self) -> str: + """SHA-256 hex digest of the canonical bytes.""" + return hashlib.sha256(self.to_bytes()).hexdigest() + + +def _month_of(value: object) -> str: + """``YYYY-MM`` prefix of a date/timestamp string (``day`` or ``first_ts``).""" + text = str(value) + return text[:7] if len(text) >= 7 else "unknown" + + +def build_shards(conn: sqlite3.Connection) -> list[Shard]: + """Build every current mart shard from *conn*, re-keyed to ``(provider, slug)``. + + Read-only: touches only the mart tables (and ``projects`` for the re-key). + Never reads or writes ``usage_events`` / ``price_book`` / raw transcripts. + """ + shards: list[Shard] = [] + for spec in _SPECS: + rows = [tuple(r) for r in conn.execute(spec.sql).fetchall()] + if spec.month_column is None: + if rows: + shards.append(Shard(spec.family, "all", spec.columns, tuple(rows))) + continue + month_idx = spec.columns.index(spec.month_column) + by_month: dict[str, list[tuple]] = defaultdict(list) + for row in rows: + by_month[_month_of(row[month_idx])].append(row) + for month in sorted(by_month): + shards.append(Shard(spec.family, month, spec.columns, tuple(by_month[month]))) + return shards + + +def shard_from_bytes(data: bytes) -> Shard: + """Inverse of :meth:`Shard.to_bytes` — reconstruct a shard from canonical bytes.""" + obj = json.loads(data) + return Shard( + family=obj["family"], + month=obj["month"], + columns=tuple(obj["columns"]), + rows=tuple(tuple(row) for row in obj["rows"]), + ) diff --git a/tests/stackunderflow/store/test_migration_v028.py b/tests/stackunderflow/store/test_migration_v028.py new file mode 100644 index 00000000..3cf6499b --- /dev/null +++ b/tests/stackunderflow/store/test_migration_v028.py @@ -0,0 +1,137 @@ +"""v028 migration: ``sync_identity`` + ``sync_outbox`` (opt-in multi-device sync). + +v028 adds two additive tables for Phase 1 of #100. These tests pin its +guarantees: + + 1. Both tables exist with the spec's columns after ``schema.apply``. + 2. ``schema.apply`` bumps ``PRAGMA user_version`` to the current head (>=28). + 3. The migration is **purely additive** — applied to a genuine v27 store it + adds ONLY ``sync_identity`` + ``sync_outbox`` and alters no existing table + (the "sync-off store is byte-identical" invariant). + 4. ``sync_identity`` is single-row (CHECK id = 1). + 5. The loader is reentrant (partial prior run recovers via ``_ADD_COLUMN_GUARDS``). +""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +from stackunderflow.store import db, schema + + +def _tables(conn) -> set[str]: + rows = conn.execute( + "SELECT name FROM sqlite_master WHERE type IN ('table', 'view')" + ).fetchall() + return {r["name"] for r in rows} + + +def _columns(conn, table: str) -> set[str]: + return {r["name"] for r in conn.execute(f"PRAGMA table_info({table})").fetchall()} + + +def _apply_up_to(conn: sqlite3.Connection, target: int) -> None: + for version, path in schema._discover(): + if version > target: + break + if path.suffix == ".sql": + conn.executescript(path.read_text()) + else: + schema._run_python_migration(conn, version, path) + + +def test_v028_creates_sync_tables(tmp_path: Path) -> None: + conn = db.connect(tmp_path / "store.db") + try: + schema.apply(conn) + tables = _tables(conn) + assert "sync_identity" in tables + assert "sync_outbox" in tables + assert _columns(conn, "sync_identity") == { + "id", "device_uuid", "key_fingerprint", "bucket_url", + "endpoint_url", "layout_version", "created_at", + } + assert _columns(conn, "sync_outbox") == { + "shard_key", "content_hash", "generation", "dirty", + "last_pushed_hash", "last_pushed_ts", + } + finally: + conn.close() + + +def test_v028_user_version_bumped(tmp_path: Path) -> None: + conn = db.connect(tmp_path / "store.db") + try: + schema.apply(conn) + assert conn.execute("PRAGMA user_version").fetchone()[0] == schema.CURRENT_VERSION + assert schema.CURRENT_VERSION >= 28 + finally: + conn.close() + + +def test_v028_is_purely_additive_on_a_genuine_v27_store(tmp_path: Path) -> None: + """Apply the real chain to v27, snapshot the schema, then apply v28 and prove + it adds ONLY the two sync tables and changes no existing table.""" + conn = db.connect(tmp_path / "store.db") + try: + _apply_up_to(conn, 27) + assert conn.execute("PRAGMA user_version").fetchone()[0] == 27 + before_tables = _tables(conn) + before_cols = {t: _columns(conn, t) for t in before_tables} + + schema.apply(conn) # v27 → head + + assert conn.execute("PRAGMA user_version").fetchone()[0] == schema.CURRENT_VERSION + after_tables = _tables(conn) + # Exactly the two new tables were added. + assert after_tables - before_tables == {"sync_identity", "sync_outbox"} + # Every pre-existing table kept its exact column set (nothing altered). + for table, cols in before_cols.items(): + assert _columns(conn, table) == cols, f"{table} columns changed" + finally: + conn.close() + + +def test_v028_sync_identity_is_single_row(tmp_path: Path) -> None: + conn = db.connect(tmp_path / "store.db") + try: + schema.apply(conn) + conn.execute( + "INSERT INTO sync_identity (id, device_uuid, key_fingerprint, bucket_url, created_at) " + "VALUES (1, 'd', 'fp', 's3://b', 't')" + ) + try: + conn.execute( + "INSERT INTO sync_identity (id, device_uuid, key_fingerprint, bucket_url, created_at) " + "VALUES (2, 'd2', 'fp2', 's3://b2', 't2')" + ) + raise AssertionError("expected CHECK (id = 1) to reject a second row") + except sqlite3.IntegrityError: + pass + finally: + conn.close() + + +def test_v028_reentrant_when_tables_already_exist(tmp_path: Path) -> None: + """Partial-application recovery: tables present but ``user_version`` behind.""" + conn = db.connect(tmp_path / "store.db") + try: + schema.apply(conn) # full chain → tables now exist + conn.execute("PRAGMA user_version = 27") # rewind so v028 looks pending + conn.commit() + schema.apply(conn) # must not raise "table already exists" + assert conn.execute("PRAGMA user_version").fetchone()[0] == schema.CURRENT_VERSION + assert {"sync_identity", "sync_outbox"}.issubset(_tables(conn)) + finally: + conn.close() + + +def test_v028_idempotent_reapply(tmp_path: Path) -> None: + conn = db.connect(tmp_path / "store.db") + try: + schema.apply(conn) + schema.apply(conn) # second call must not raise + assert conn.execute("PRAGMA user_version").fetchone()[0] == schema.CURRENT_VERSION + finally: + conn.close() diff --git a/tests/stackunderflow/sync/__init__.py b/tests/stackunderflow/sync/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/stackunderflow/sync/conftest.py b/tests/stackunderflow/sync/conftest.py new file mode 100644 index 00000000..ece2a1da --- /dev/null +++ b/tests/stackunderflow/sync/conftest.py @@ -0,0 +1,130 @@ +"""Shared fixtures for the sync tests. + +``make_store`` returns freshly-migrated stores (so a test can build two +independent "devices"); ``seed_marts`` populates the Overview/Cost-core marts +with a small, deterministic dataset. Both are dependency-free — the crypto-path +tests layer ``pyrage`` on top via ``importorskip``. +""" + +from __future__ import annotations + +import sqlite3 + +import pytest + +from stackunderflow.store import db, schema + +_DAILY = ( + # day, project_id, provider, model, speed, in, out, cr, cc, msg, sess, cost + ("2026-07-01", "opus", "standard", 100, 50, 10, 5, 3, 1, 1.5), + ("2026-07-02", "opus", "standard", 200, 80, 20, 8, 4, 1, 2.5), +) +_BETA_DAILY = ("2026-06-30", "sonnet", "standard", 40, 20, 4, 2, 2, 1, 0.4) + + +def _seed(conn: sqlite3.Connection, *, alpha_id: int = 1, beta_id: int = 2, + session_id: str = "sess-a", scale: int = 1) -> None: + """Insert a deterministic mart dataset. ``scale`` multiplies token/cost measures. + + ``alpha_id`` / ``beta_id`` are the machine-local ``project_id`` values — vary + them across "devices" to exercise re-keying; keep the slugs (``alpha`` / + ``beta``) fixed so the stable ``(provider, slug)`` identity matches. + """ + conn.execute( + "INSERT INTO projects (id, provider, slug, path, display_name, first_seen, last_modified) " + "VALUES (?, 'claude', 'alpha', '/Users/x/alpha', 'Alpha', 0, 0)", + (alpha_id,), + ) + conn.execute( + "INSERT INTO projects (id, provider, slug, path, display_name, first_seen, last_modified) " + "VALUES (?, 'claude', 'beta', '/Users/x/beta', 'Beta', 0, 0)", + (beta_id,), + ) + + for day, model, speed, tin, tout, cr, cc, msg, sess, cost in _DAILY: + conn.execute( + "INSERT INTO daily_mart (day, project_id, provider, model, speed, " + "input_tokens, output_tokens, cache_read, cache_create, message_count, " + "session_count, cost_usd) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + (day, alpha_id, "claude", model, speed, tin * scale, tout * scale, + cr * scale, cc * scale, msg, sess, cost * scale), + ) + day, model, speed, tin, tout, cr, cc, msg, sess, cost = _BETA_DAILY + conn.execute( + "INSERT INTO daily_mart (day, project_id, provider, model, speed, " + "input_tokens, output_tokens, cache_read, cache_create, message_count, " + "session_count, cost_usd) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + (day, beta_id, "claude", model, speed, tin * scale, tout * scale, + cr * scale, cc * scale, msg, sess, cost * scale), + ) + + conn.execute( + "INSERT INTO project_mart (project_id, provider, slug, display_name, first_ts, " + "last_ts, total_messages, total_sessions, total_input_tokens, total_output_tokens, " + "total_cache_read, total_cache_create, total_cost_usd) " + "VALUES (?, 'claude', 'alpha', 'Alpha', '2026-07-01T10:00:00', '2026-07-02T12:00:00', " + "7, 2, ?, ?, ?, ?, ?)", + (alpha_id, 300 * scale, 130 * scale, 30 * scale, 13 * scale, 4.0 * scale), + ) + conn.execute( + "INSERT INTO project_mart (project_id, provider, slug, display_name, first_ts, " + "last_ts, total_messages, total_sessions, total_input_tokens, total_output_tokens, " + "total_cache_read, total_cache_create, total_cost_usd) " + "VALUES (?, 'claude', 'beta', 'Beta', '2026-06-30T09:00:00', '2026-06-30T10:00:00', " + "2, 1, ?, ?, ?, ?, ?)", + (beta_id, 40 * scale, 20 * scale, 4 * scale, 2 * scale, 0.4 * scale), + ) + + conn.executemany( + "INSERT INTO provider_day_mart (day, provider, cost_usd, message_count, " + "session_count, project_count) VALUES (?,?,?,?,?,?)", + [ + ("2026-07-01", "claude", 1.5 * scale, 3, 1, 1), + ("2026-06-30", "claude", 0.4 * scale, 2, 1, 1), + ], + ) + conn.executemany( + "INSERT INTO model_day_mart (day, model, speed, cost_usd, input_tokens, " + "output_tokens, cache_read, cache_create, message_count, session_count) " + "VALUES (?,?,?,?,?,?,?,?,?,?)", + [ + ("2026-07-01", "opus", "standard", 1.5 * scale, 100 * scale, 50 * scale, + 10 * scale, 5 * scale, 3, 1), + ("2026-06-30", "sonnet", "standard", 0.4 * scale, 40 * scale, 20 * scale, + 4 * scale, 2 * scale, 2, 1), + ], + ) + + conn.execute( + "INSERT INTO session_mart (session_id, project_id, provider, primary_model, " + "first_ts, last_ts, message_count, user_message_count, assistant_message_count, " + "input_tokens, output_tokens, cache_read, cache_create, cost_usd, is_one_shot, cwd) " + "VALUES (?, ?, 'claude', 'opus', '2026-07-01T10:00:00', '2026-07-01T11:00:00', " + "5, 2, 3, ?, ?, ?, ?, ?, 0, '/Users/x/alpha')", + (session_id, alpha_id, 300 * scale, 130 * scale, 30 * scale, 13 * scale, 4.0 * scale), + ) + + +@pytest.fixture +def make_store(tmp_path): + """Factory returning fresh, migrated stores (call once per simulated device).""" + counter = {"n": 0} + + def _make() -> sqlite3.Connection: + counter["n"] += 1 + conn = db.connect(tmp_path / f"store{counter['n']}.db") + schema.apply(conn) + return conn + + return _make + + +@pytest.fixture +def store_conn(make_store): + return make_store() + + +@pytest.fixture +def seed_marts(): + """Return the ``_seed`` helper (so tests can seed several stores).""" + return _seed diff --git a/tests/stackunderflow/sync/test_bucket.py b/tests/stackunderflow/sync/test_bucket.py new file mode 100644 index 00000000..1e2c09cb --- /dev/null +++ b/tests/stackunderflow/sync/test_bucket.py @@ -0,0 +1,75 @@ +"""ObjectStore fake + URL parsing — dependency-free (no boto3 needed).""" + +from __future__ import annotations + +import pytest + +from stackunderflow.sync import bucket + + +def test_put_get_roundtrip(): + store = bucket.InMemoryObjectStore() + store.put("a/b.age", b"hello") + assert store.get("a/b.age") == b"hello" + assert "a/b.age" in store + assert len(store) == 1 + + +def test_get_missing_raises_object_not_found(): + store = bucket.InMemoryObjectStore() + with pytest.raises(bucket.ObjectNotFound): + store.get("nope") + + +def test_list_filters_by_prefix_and_sorts(): + store = bucket.InMemoryObjectStore() + store.put("p/1", b"1") + store.put("p/3", b"3") + store.put("p/2", b"2") + store.put("other/x", b"x") + assert store.list("p/") == ["p/1", "p/2", "p/3"] + assert store.list("other/") == ["other/x"] + assert store.list("zzz") == [] + + +def test_delete_is_idempotent(): + store = bucket.InMemoryObjectStore() + store.put("k", b"v") + store.delete("k") + store.delete("k") # no error on a second delete + assert "k" not in store + + +def test_call_counts_track_operations(): + store = bucket.InMemoryObjectStore() + store.put("k", b"v") + store.get("k") + store.list("k") + store.delete("k") + assert store.put_calls == 1 + assert store.get_calls == 1 + assert store.list_calls == 1 + assert store.delete_calls == 1 + + +def test_in_memory_satisfies_protocol(): + assert isinstance(bucket.InMemoryObjectStore(), bucket.ObjectStore) + + +@pytest.mark.parametrize( + ("url", "expected"), + [ + ("s3://my-bucket", ("my-bucket", "")), + ("s3://my-bucket/", ("my-bucket", "")), + ("s3://my-bucket/some/prefix", ("my-bucket", "some/prefix")), + ("s3://my-bucket/some/prefix/", ("my-bucket", "some/prefix")), + ], +) +def test_parse_bucket_url(url, expected): + assert bucket.parse_bucket_url(url) == expected + + +@pytest.mark.parametrize("bad", ["my-bucket", "https://x", "s3://"]) +def test_parse_bucket_url_rejects_bad(bad): + with pytest.raises(ValueError): + bucket.parse_bucket_url(bad) diff --git a/tests/stackunderflow/sync/test_cipher.py b/tests/stackunderflow/sync/test_cipher.py new file mode 100644 index 00000000..e1f66bff --- /dev/null +++ b/tests/stackunderflow/sync/test_cipher.py @@ -0,0 +1,32 @@ +"""age encrypt/decrypt roundtrip + failure modes. Gated on pyrage.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("pyrage") + +from stackunderflow.sync import cipher, keys # noqa: E402 - after importorskip + + +def test_encrypt_decrypt_roundtrip(): + identity = keys.generate_identity() + ct = cipher.encrypt(b"the quick brown fox", identity.recipient) + assert ct != b"the quick brown fox" # actually encrypted + assert cipher.decrypt(ct, identity.secret) == b"the quick brown fox" + + +def test_wrong_key_raises_clean_decrypt_error(): + owner = keys.generate_identity() + other = keys.generate_identity() + ct = cipher.encrypt(b"secret aggregate", owner.recipient) + with pytest.raises(cipher.DecryptError): + cipher.decrypt(ct, other.secret) + + +def test_tampered_ciphertext_raises(): + identity = keys.generate_identity() + ct = bytearray(cipher.encrypt(b"payload bytes here", identity.recipient)) + ct[-1] ^= 0xFF # flip the last byte + with pytest.raises(cipher.DecryptError): + cipher.decrypt(bytes(ct), identity.secret) diff --git a/tests/stackunderflow/sync/test_cli_sync.py b/tests/stackunderflow/sync/test_cli_sync.py new file mode 100644 index 00000000..8eb48459 --- /dev/null +++ b/tests/stackunderflow/sync/test_cli_sync.py @@ -0,0 +1,153 @@ +"""``stackunderflow sync {init,push,status}`` CLI surface. + +Store + state dir are redirected to ``tmp_path`` (as in ``cli/test_doctor.py``), +so nothing touches ``~/.stackunderflow``. Crypto-dependent flows are gated on +``pyrage``; the deps-missing hint and the ``status``/``not-configured`` paths run +without any optional dependency (``_sync_missing_deps`` is monkeypatched so the +result is independent of what's actually installed). +""" + +from __future__ import annotations + +import json +import os +import stat + +import pytest +from click.testing import CliRunner + +import stackunderflow.cli as cli_mod +import stackunderflow.deps as deps +from stackunderflow.cli import cli +from stackunderflow.store import db, schema + + +def _prep(tmp_path, monkeypatch): + store = tmp_path / "store.db" + conn = db.connect(store) + schema.apply(conn) + conn.close() + state = tmp_path / "state" + state.mkdir() + monkeypatch.setattr(deps, "store_path", store) + monkeypatch.setattr(cli_mod, "_STATE_DIR", state) + return store, state + + +def _fingerprint(store): + conn = db.connect(store) + try: + row = conn.execute("SELECT key_fingerprint FROM sync_identity WHERE id = 1").fetchone() + return row["key_fingerprint"] if row else None + finally: + conn.close() + + +def test_status_off_text(tmp_path, monkeypatch): + _prep(tmp_path, monkeypatch) + r = CliRunner().invoke(cli, ["sync", "status"]) + assert r.exit_code == 0, r.output + assert "Sync: off" in r.output + + +def test_status_off_json(tmp_path, monkeypatch): + _prep(tmp_path, monkeypatch) + r = CliRunner().invoke(cli, ["sync", "status", "--json"]) + assert r.exit_code == 0, r.output + payload = json.loads(r.output) + assert payload["enabled"] is False + assert payload["pending_count"] == 0 + + +def test_init_missing_deps_prints_hint(tmp_path, monkeypatch): + _prep(tmp_path, monkeypatch) + monkeypatch.setattr(cli_mod, "_sync_missing_deps", lambda **_: ["pyrage"]) + r = CliRunner().invoke(cli, ["sync", "init", "--bucket", "s3://b"]) + assert r.exit_code == 1 + assert "pip install 'stackunderflow[sync]'" in r.output + + +def test_push_not_configured_exits_nonzero(tmp_path, monkeypatch): + _prep(tmp_path, monkeypatch) + monkeypatch.setattr(cli_mod, "_sync_missing_deps", lambda **_: []) # pretend deps present + r = CliRunner().invoke(cli, ["sync", "push"]) + assert r.exit_code == 1 + assert "not configured" in r.output + + +def test_init_generates_key_and_configures(tmp_path, monkeypatch): + pytest.importorskip("pyrage") + store, state = _prep(tmp_path, monkeypatch) + + r = CliRunner().invoke(cli, ["sync", "init", "--bucket", "s3://my-bucket"]) + assert r.exit_code == 0, r.output + # Loud zero-knowledge / key-loss banner, and the key shown once. + assert "UNRECOVERABLE" in r.output + assert "AGE-SECRET-KEY-1" in r.output + + keyfile = state / "sync-identity" + assert keyfile.exists() + assert stat.S_IMODE(os.stat(keyfile).st_mode) == 0o600 + + conn = db.connect(store) + row = conn.execute( + "SELECT device_uuid, key_fingerprint, bucket_url FROM sync_identity WHERE id = 1" + ).fetchone() + conn.close() + assert row["bucket_url"] == "s3://my-bucket" + # Only the fingerprint is persisted — never the secret. + assert row["key_fingerprint"] + assert "AGE-SECRET-KEY" not in row["key_fingerprint"] + + r2 = CliRunner().invoke(cli, ["sync", "status"]) + assert "Sync: on" in r2.output + assert row["device_uuid"] in r2.output + + +def test_init_refuses_second_without_force_then_rotates_with_force(tmp_path, monkeypatch): + pytest.importorskip("pyrage") + store, _ = _prep(tmp_path, monkeypatch) + + assert CliRunner().invoke(cli, ["sync", "init", "--bucket", "s3://b"]).exit_code == 0 + first_fp = _fingerprint(store) + + r = CliRunner().invoke(cli, ["sync", "init", "--bucket", "s3://b"]) + assert r.exit_code == 1 + assert "already configured" in r.output + assert _fingerprint(store) == first_fp # unchanged + + r2 = CliRunner().invoke(cli, ["sync", "init", "--bucket", "s3://b", "--force"]) + assert r2.exit_code == 0, r2.output + assert _fingerprint(store) != first_fp # key rotated + + +def test_push_end_to_end_with_fake_bucket(tmp_path, monkeypatch, seed_marts): + pytest.importorskip("pyrage") + store, state = _prep(tmp_path, monkeypatch) + conn = db.connect(store) + seed_marts(conn) + conn.close() + + assert CliRunner().invoke(cli, ["sync", "init", "--bucket", "s3://b"]).exit_code == 0 + + from stackunderflow.sync import bucket as bkt + + fake = bkt.InMemoryObjectStore() + monkeypatch.setattr("stackunderflow.sync.bucket.s3_store_from_url", lambda *a, **k: fake) + monkeypatch.setattr("stackunderflow.sync.keys._read_keychain", lambda service=None: None) + monkeypatch.setattr(cli_mod, "_sync_missing_deps", lambda **_: []) + + r = CliRunner().invoke(cli, ["sync", "push"]) + assert r.exit_code == 0, r.output + assert "Pushed" in r.output + assert len(fake) > 0 + assert any(k.endswith("manifest.age") for k in fake.list("")) + + # Idempotent second push — nothing changed. + r2 = CliRunner().invoke(cli, ["sync", "push"]) + assert r2.exit_code == 0, r2.output + assert "Up to date" in r2.output + + # status now shows no pending. + r3 = CliRunner().invoke(cli, ["sync", "status"]) + assert "pending upload: 0" in r3.output diff --git a/tests/stackunderflow/sync/test_keys.py b/tests/stackunderflow/sync/test_keys.py new file mode 100644 index 00000000..bf4e1926 --- /dev/null +++ b/tests/stackunderflow/sync/test_keys.py @@ -0,0 +1,68 @@ +"""Key resolution / storage / fingerprint. The resolution + file legs need no +optional dependency; identity generation is gated on pyrage.""" + +from __future__ import annotations + +import os +import stat + +import pytest + +from stackunderflow.sync import keys + + +def test_fingerprint_is_stable_short_hex(): + fp = keys.fingerprint("age1exampledummyrecipient") + assert fp == keys.fingerprint("age1exampledummyrecipient") + assert len(fp) == 16 + assert all(c in "0123456789abcdef" for c in fp) + assert keys.fingerprint("age1other") != fp + + +def test_resolve_secret_env_wins(tmp_path): + keys.store_secret_file("FILE-KEY", tmp_path) + got = keys.resolve_secret( + tmp_path, + env={keys.ENV_KEY: "ENV-KEY"}, + keychain_reader=lambda: None, + ) + assert got == "ENV-KEY" + + +def test_resolve_secret_keychain_before_file(tmp_path): + keys.store_secret_file("FILE-KEY", tmp_path) + got = keys.resolve_secret( + tmp_path, + env={}, + keychain_reader=lambda: "KEYCHAIN-KEY", + ) + assert got == "KEYCHAIN-KEY" + + +def test_resolve_secret_falls_back_to_file(tmp_path): + keys.store_secret_file("FILE-KEY", tmp_path) + got = keys.resolve_secret(tmp_path, env={}, keychain_reader=lambda: None) + assert got == "FILE-KEY" + + +def test_resolve_secret_none_when_absent(tmp_path): + got = keys.resolve_secret(tmp_path, env={}, keychain_reader=lambda: None) + assert got is None + + +def test_store_secret_file_is_0600(tmp_path): + path = keys.store_secret_file("AGE-SECRET-KEY-1-EXAMPLE", tmp_path) + assert path == keys.identity_path(tmp_path) + mode = stat.S_IMODE(os.stat(path).st_mode) + assert mode == 0o600 + assert path.read_text().strip() == "AGE-SECRET-KEY-1-EXAMPLE" + + +def test_generate_identity_and_recipient_for(): + pytest.importorskip("pyrage") + identity = keys.generate_identity() + assert identity.secret.startswith("AGE-SECRET-KEY-1") + assert identity.recipient.startswith("age1") + assert identity.fingerprint == keys.fingerprint(identity.recipient) + # The public recipient is recoverable from the secret alone. + assert keys.recipient_for(identity.secret) == identity.recipient diff --git a/tests/stackunderflow/sync/test_runner.py b/tests/stackunderflow/sync/test_runner.py new file mode 100644 index 00000000..2e1cd8e5 --- /dev/null +++ b/tests/stackunderflow/sync/test_runner.py @@ -0,0 +1,261 @@ +"""Push idempotency, outbox, two-phase commit, status, failure injection. + +The core :func:`runner.push` is exercised WITHOUT any optional dependency by +injecting an identity ``encryptor`` (``lambda pt: pt``) and the in-memory fake +store. The ``run_push`` crypto integration is gated on ``pyrage``. +""" + +from __future__ import annotations + +import json + +import pytest + +from stackunderflow.sync import bucket, runner, serialize + +_EXPECTED_SHARDS = 8 # daily 06/07, provider_day 06/07, model_day 06/07, project all, session 07 + + +class _ExplodingStore(bucket.InMemoryObjectStore): + """InMemory store that raises on the Nth ``put`` — for failure injection.""" + + def __init__(self, fail_on_put_number: int) -> None: + super().__init__() + self._fail_on = fail_on_put_number + self._n = 0 + + def put(self, key: str, data: bytes) -> None: + self._n += 1 + if self._n == self._fail_on: + raise RuntimeError("bucket unavailable") + super().put(key, data) + + +def _push(conn, store, **kw): + return runner.push( + conn, store, + device_uuid=kw.pop("device_uuid", "dev-1"), + key_fingerprint=kw.pop("key_fingerprint", "fp-1"), + encryptor=lambda pt: pt, + **kw, + ) + + +# ── identity ────────────────────────────────────────────────────────────────── + + +def test_write_and_load_identity(store_conn): + assert runner.load_identity(store_conn) is None + assert runner.is_enabled(store_conn) is False + runner.write_identity( + store_conn, device_uuid="d1", key_fingerprint="fp", + bucket_url="s3://b", endpoint_url="https://e", created_at="t", + ) + ident = runner.load_identity(store_conn) + assert ident["device_uuid"] == "d1" + assert ident["endpoint_url"] == "https://e" + assert runner.is_enabled(store_conn) is True + + +# ── push ────────────────────────────────────────────────────────────────────── + + +def test_push_uploads_all_shards_and_commits_manifest(store_conn, seed_marts): + seed_marts(store_conn) + store = bucket.InMemoryObjectStore() + result = _push(store_conn, store, device_uuid="dev-xyz") + + assert result.uploaded == _EXPECTED_SHARDS + assert result.skipped == 0 + assert result.generation == 1 + assert result.manifest_written is True + # object layout: readable per-device keys under stackunderflow/v1// + all_keys = store.list("") + assert "stackunderflow/v1/dev-xyz/manifest.age" in all_keys + assert "stackunderflow/v1/dev-xyz/shards/daily_mart.2026-07.age" in all_keys + assert "stackunderflow/v1/dev-xyz/shards/project_mart.all.age" in all_keys + # never the excluded / raw surfaces + assert not any( + tok in k for k in all_keys + for tok in ("usage_events", "price_book", "message_tool") + ) + + +def test_manifest_records_every_shard(store_conn, seed_marts): + seed_marts(store_conn) + store = bucket.InMemoryObjectStore() + _push(store_conn, store, device_uuid="dev-xyz") + + # identity encryptor ⇒ the manifest object is plain JSON bytes. + manifest = json.loads(store.get("stackunderflow/v1/dev-xyz/manifest.age")) + assert manifest["schema"] == "stackunderflow.sync/1" + assert manifest["device_uuid"] == "dev-xyz" + assert manifest["generation"] == 1 + assert set(manifest["shards"]) == {s.shard_key for s in serialize.build_shards(store_conn)} + entry = manifest["shards"]["daily_mart.2026-07"] + assert entry["object_key"] == "stackunderflow/v1/dev-xyz/shards/daily_mart.2026-07.age" + assert len(entry["content_hash"]) == 64 + assert entry["bytes"] > 0 + + +def test_push_is_idempotent_zero_puts_when_unchanged(store_conn, seed_marts): + seed_marts(store_conn) + store = bucket.InMemoryObjectStore() + _push(store_conn, store) + puts_after_first = store.put_calls # 8 shards + 1 manifest + + result = _push(store_conn, store) + assert result.uploaded == 0 + assert result.skipped == _EXPECTED_SHARDS + assert result.manifest_written is False + assert store.put_calls == puts_after_first # ZERO new puts, incl. manifest + + +def test_push_reuploads_only_changed_shard(store_conn, seed_marts): + seed_marts(store_conn) + store = bucket.InMemoryObjectStore() + _push(store_conn, store) + puts_after_first = store.put_calls + + # Mutate one July daily row → only daily_mart.2026-07's hash changes. + store_conn.execute("UPDATE daily_mart SET cost_usd = cost_usd + 1 WHERE day = '2026-07-01'") + result = _push(store_conn, store) + + assert result.uploaded == 1 + assert result.shard_keys == ["daily_mart.2026-07"] + assert result.skipped == _EXPECTED_SHARDS - 1 + assert result.generation == 2 + assert result.manifest_written is True + # exactly one shard + one manifest re-put + assert store.put_calls - puts_after_first == 2 + + +def test_push_never_mutates_source_marts(store_conn, seed_marts): + seed_marts(store_conn) + before = store_conn.execute("SELECT count(*), total(cost_usd) FROM daily_mart").fetchone() + _push(store_conn, bucket.InMemoryObjectStore()) + after = store_conn.execute("SELECT count(*), total(cost_usd) FROM daily_mart").fetchone() + assert tuple(before) == tuple(after) # push is read-only on the marts + + +def test_push_failure_does_not_advance_outbox_or_commit_manifest(store_conn, seed_marts): + seed_marts(store_conn) + # Fail on the 2nd put: the 1st shard is uploaded+recorded, then we blow up + # before the manifest — a reader keeps the previous (here: empty) manifest. + store = _ExplodingStore(fail_on_put_number=2) + with pytest.raises(RuntimeError): + _push(store_conn, store) + + outbox = store_conn.execute("SELECT shard_key, last_pushed_hash FROM sync_outbox").fetchall() + assert len(outbox) == 1 # only the one shard that uploaded before the failure + assert outbox[0]["last_pushed_hash"] is not None + assert not any("manifest" in k for k in store.list("")) # phase 2 never reached + + # A retry against a healthy store finishes the job (already-pushed shard skipped). + good = bucket.InMemoryObjectStore() + result = _push(store_conn, good) + assert result.uploaded == _EXPECTED_SHARDS - 1 + assert result.skipped == 1 + assert result.manifest_written is True + + +# ── status ──────────────────────────────────────────────────────────────────── + + +def test_status_disabled(store_conn): + st = runner.status(store_conn) + assert st.enabled is False + assert st.pending == [] + assert st.as_dict()["enabled"] is False + + +def test_status_enabled_reports_pending(store_conn, seed_marts): + seed_marts(store_conn) + runner.write_identity( + store_conn, device_uuid="d1", key_fingerprint="fp", + bucket_url="s3://b", endpoint_url=None, created_at="t", + ) + st = runner.status(store_conn) + assert st.enabled is True + assert st.device_uuid == "d1" + assert st.bucket_url == "s3://b" + assert len(st.pending) == _EXPECTED_SHARDS # nothing pushed yet + assert st.last_push_ts is None + + +def test_status_after_push_has_no_pending(store_conn, seed_marts): + seed_marts(store_conn) + runner.write_identity( + store_conn, device_uuid="d1", key_fingerprint="fp", + bucket_url="s3://b", endpoint_url=None, created_at="t", + ) + runner.push( + store_conn, bucket.InMemoryObjectStore(), + device_uuid="d1", key_fingerprint="fp", + encryptor=lambda pt: pt, now="2026-07-03T00:00:00+00:00", + ) + st = runner.status(store_conn) + assert st.pending == [] + assert st.last_push_ts == "2026-07-03T00:00:00+00:00" + + +# ── run_push (deps wiring) ──────────────────────────────────────────────────── + + +def test_run_push_not_configured(store_conn, tmp_path): + with pytest.raises(runner.SyncNotConfigured): + runner.run_push(store_conn, state_dir=tmp_path, store=bucket.InMemoryObjectStore(), env={}) + + +def test_run_push_key_missing(store_conn, seed_marts, tmp_path, monkeypatch): + seed_marts(store_conn) + runner.write_identity( + store_conn, device_uuid="dz", key_fingerprint="fp", + bucket_url="s3://b", endpoint_url=None, created_at="t", + ) + monkeypatch.setattr("stackunderflow.sync.keys._read_keychain", lambda service=None: None) + with pytest.raises(runner.SyncKeyMissing): + runner.run_push(store_conn, state_dir=tmp_path, store=bucket.InMemoryObjectStore(), env={}) + + +def test_run_push_with_real_crypto(store_conn, seed_marts, tmp_path, monkeypatch): + pytest.importorskip("pyrage") + from stackunderflow.sync import cipher, keys + + seed_marts(store_conn) + identity = keys.generate_identity() + keys.store_secret_file(identity.secret, tmp_path) + runner.write_identity( + store_conn, device_uuid="dz", key_fingerprint=identity.fingerprint, + bucket_url="s3://b", endpoint_url=None, created_at="t", + ) + monkeypatch.setattr("stackunderflow.sync.keys._read_keychain", lambda service=None: None) + store = bucket.InMemoryObjectStore() + + result = runner.run_push(store_conn, state_dir=tmp_path, store=store, env={}) + assert result.uploaded == _EXPECTED_SHARDS + assert result.manifest_written is True + + # The object really is age-encrypted and decrypts back to the serialized shard. + ct = store.get("stackunderflow/v1/dz/shards/daily_mart.2026-07.age") + assert ct.startswith(b"age-encryption.org/") or b"age" in ct[:32] + recovered = serialize.shard_from_bytes(cipher.decrypt(ct, identity.secret)) + local = {s.shard_key: s for s in serialize.build_shards(store_conn)} + assert recovered.content_hash == local["daily_mart.2026-07"].content_hash + + +def test_run_push_key_mismatch(store_conn, seed_marts, tmp_path, monkeypatch): + pytest.importorskip("pyrage") + from stackunderflow.sync import keys + + seed_marts(store_conn) + right = keys.generate_identity() + wrong = keys.generate_identity() + keys.store_secret_file(wrong.secret, tmp_path) # the file holds the WRONG key + runner.write_identity( + store_conn, device_uuid="dz", key_fingerprint=right.fingerprint, + bucket_url="s3://b", endpoint_url=None, created_at="t", + ) + monkeypatch.setattr("stackunderflow.sync.keys._read_keychain", lambda service=None: None) + with pytest.raises(runner.SyncKeyMismatch): + runner.run_push(store_conn, state_dir=tmp_path, store=bucket.InMemoryObjectStore(), env={}) diff --git a/tests/stackunderflow/sync/test_serialize.py b/tests/stackunderflow/sync/test_serialize.py new file mode 100644 index 00000000..1372e104 --- /dev/null +++ b/tests/stackunderflow/sync/test_serialize.py @@ -0,0 +1,133 @@ +"""Serialization, determinism, and re-keying — all dependency-free (no pyrage).""" + +from __future__ import annotations + +from stackunderflow.sync import serialize + + +def _by_key(shards): + return {s.shard_key: s for s in shards} + + +def test_build_shards_families_and_months(store_conn, seed_marts): + seed_marts(store_conn) + shards = _by_key(serialize.build_shards(store_conn)) + + # The five Overview/Cost-core families, sharded by month where dated. + assert set(shards) == { + "daily_mart.2026-07", "daily_mart.2026-06", + "provider_day_mart.2026-07", "provider_day_mart.2026-06", + "model_day_mart.2026-07", "model_day_mart.2026-06", + "project_mart.all", + "session_mart.2026-07", + } + # message_tool_mart (file_path) is NEVER a shard family. + assert not any("message_tool" in k for k in shards) + # July daily has the two alpha rows; June has the one beta row. + assert len(shards["daily_mart.2026-07"].rows) == 2 + assert len(shards["daily_mart.2026-06"].rows) == 1 + + +def test_mart_families_are_exactly_the_core_five(): + assert set(serialize.MART_FAMILIES) == { + "daily_mart", "project_mart", "provider_day_mart", "model_day_mart", "session_mart", + } + # Never usage_events / price_book / message_tool_mart. + for forbidden in ("usage_events", "price_book", "message_tool_mart"): + assert forbidden not in serialize.MART_FAMILIES + + +def test_rekey_drops_project_id_and_cwd(store_conn, seed_marts): + seed_marts(store_conn) + shards = _by_key(serialize.build_shards(store_conn)) + + daily = shards["daily_mart.2026-07"] + assert "provider" in daily.columns and "slug" in daily.columns + assert "project_id" not in daily.columns + + session = shards["session_mart.2026-07"] + assert "slug" in session.columns + assert "project_id" not in session.columns + # cwd (a filesystem path) is deliberately excluded — never on the wire. + assert "cwd" not in session.columns + assert b"/Users/x/alpha" not in session.to_bytes() + + +def test_serialization_is_deterministic(store_conn, seed_marts): + seed_marts(store_conn) + first = {s.shard_key: s.content_hash for s in serialize.build_shards(store_conn)} + second = {s.shard_key: s.content_hash for s in serialize.build_shards(store_conn)} + assert first == second + + +def test_roundtrip_bytes(store_conn, seed_marts): + seed_marts(store_conn) + for shard in serialize.build_shards(store_conn): + assert serialize.shard_from_bytes(shard.to_bytes()) == shard + + +def test_rekey_identical_across_devices_with_different_local_ids(make_store, seed_marts): + """Device A (ids 1,2) and B (ids 41,42) with the same (provider, slug) data + produce IDENTICAL shard content-hashes — proving the local project_id was + re-keyed out and (provider, slug) is the stable cross-device identity.""" + dev_a = make_store() + dev_b = make_store() + seed_marts(dev_a, alpha_id=1, beta_id=2, session_id="s1", scale=1) + seed_marts(dev_b, alpha_id=41, beta_id=42, session_id="s1", scale=1) + + a = {s.shard_key: s.content_hash for s in serialize.build_shards(dev_a)} + b = {s.shard_key: s.content_hash for s in serialize.build_shards(dev_b)} + + for key in ("daily_mart.2026-07", "project_mart.all", "session_mart.2026-07"): + assert a[key] == b[key], f"{key} should be device-independent after re-key" + + +def test_different_slug_does_not_merge(make_store, seed_marts): + """Same logical project at a different path → different slug → distinct row (§5.2).""" + dev_a = make_store() + dev_c = make_store() + seed_marts(dev_a, alpha_id=1, beta_id=2) + seed_marts(dev_c, alpha_id=1, beta_id=2) + # Rename alpha's slug on device C. + dev_c.execute("UPDATE project_mart SET slug='alpha2' WHERE slug='alpha'") + dev_c.execute("UPDATE projects SET slug='alpha2' WHERE slug='alpha'") + + a = {s.shard_key: s.content_hash for s in serialize.build_shards(dev_a)} + c = {s.shard_key: s.content_hash for s in serialize.build_shards(dev_c)} + assert a["project_mart.all"] != c["project_mart.all"] + assert a["daily_mart.2026-07"] != c["daily_mart.2026-07"] + + +def test_union_sums_at_stable_grain(make_store, seed_marts): + """Two devices' re-keyed daily rows SUM at (day, provider, slug, model, speed).""" + dev_a = make_store() + dev_b = make_store() + seed_marts(dev_a, alpha_id=1, beta_id=2, session_id="a", scale=1) + seed_marts(dev_b, alpha_id=7, beta_id=8, session_id="b", scale=3) + + def july_daily(conn): + shard = _by_key(serialize.build_shards(conn))["daily_mart.2026-07"] + cols = shard.columns + return { + (r[cols.index("day")], r[cols.index("slug")]): r[cols.index("input_tokens")] + for r in shard.rows + } + + a = july_daily(dev_a) + b = july_daily(dev_b) + # Shared grain (alpha, 2026-07-01): 100 (A) + 300 (B) = 400. + key = ("2026-07-01", "alpha") + assert a[key] == 100 + assert b[key] == 300 + assert a[key] + b[key] == 400 + + +def test_serialize_never_queries_usage_events_or_price_book(store_conn, seed_marts): + """Only the five marts are produced, and no export query touches the fact + table / price book — raw usage and the rate card never reach the wire.""" + seed_marts(store_conn) + families = {s.family for s in serialize.build_shards(store_conn)} + assert families == set(serialize.MART_FAMILIES) + for spec in serialize._SPECS: + assert "usage_events" not in spec.sql + assert "price_book" not in spec.sql From 92270dab288e15e6b73a0c6bf47052ebbc51bf02 Mon Sep 17 00:00:00 2001 From: Yad Konrad Date: Fri, 3 Jul 2026 10:18:29 -0400 Subject: [PATCH 3/3] =?UTF-8?q?feat(benchmark):=20comparative=20benchmark?= =?UTF-8?q?=20engine=20MVP=20(#99)=20=E2=80=94=20stratified/CI-guarded,=20?= =?UTF-8?q?Compare-tab=20panel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/specs/benchmark-rubric-v1.md | 105 ++ .../src/components/dashboard/CompareTab.tsx | 4 + .../components/dashboard/ModelWinsPanel.tsx | 296 +++++ stackunderflow-ui/src/services/api.ts | 16 + stackunderflow-ui/src/types/api.ts | 81 ++ stackunderflow/cli.py | 217 ++++ stackunderflow/reports/benchmark.py | 1033 +++++++++++++++++ stackunderflow/routes/benchmark.py | 243 ++++ stackunderflow/server.py | 2 + stackunderflow/services/benchmark_stats.py | 312 +++++ stackunderflow/services/meta_agent.py | 71 ++ stackunderflow/services/mode_recommender.py | 73 +- stackunderflow/services/tag_service.py | 42 +- stackunderflow/services/task_classifier.py | 258 ++++ ...sTab-ZL7pzav8.js => AgentsTab-DJ-0Trzo.js} | 2 +- ...b-D5-cF5Yn.js => BookmarksTab-DAjJ72Pj.js} | 2 +- ...Tab-CukAskCj.js => BudgetsTab-n3gfgUaU.js} | 2 +- ...rhCZGhW.js => CodingHealthTab-Cz4e1hEF.js} | 2 +- ...ab-CWrSh_N3.js => CommandsTab-D4oiPDTW.js} | 2 +- .../react/assets/CompareTab-C7M_P6g3.js | 1 - .../react/assets/CompareTab-CVqkwtRX.js | 1 + ...8QfR7v.js => ContextReplayTab-DjTyR19Y.js} | 2 +- ...ostTab-DmTsx3v9.js => CostTab-BAdmE1EC.js} | 2 +- ...able-DxCemy_F.js => DataTable-D0JcWuTA.js} | 2 +- ...P-q.js => EstimatedCostMarker-DkSj8xDc.js} | 2 +- ...rBar-CV18m0Rx.js => FilterBar-DuduzIn6.js} | 2 +- ...ksTab-D6x9mT4f.js => ForksTab-ovi_pavq.js} | 2 +- ...y-DK0jJa88.js => IconActivity-BkaNd4nH.js} | 2 +- ...qIiBOhp.js => IconAlertCircle-DyBnJSK0.js} | 2 +- ...DHcdKZxB.js => IconArrowRight-xIaEMpRY.js} | 2 +- ...Up-kJ6mwirp.js => IconArrowUp-CszCUTPq.js} | 2 +- ...heck-DaUkhaCc.js => IconCheck-DwMBTi_m.js} | 2 +- ...eX-Ci-XRc_E.js => IconCircleX-1-WYCT5P.js} | 2 +- ...lock-CNOtVJvP.js => IconClock-Dl-xdEIm.js} | 2 +- ...B8eFf-3y.js => IconClockHour4-UHU_zj__.js} | 2 +- ...nCode-B8NfAF9i.js => IconCode-DaQ_eMLh.js} | 2 +- ...nCopy-BZaMjQku.js => IconCopy-DA8lvnPB.js} | 2 +- ...t-u1caIhwu.js => IconFileText-C9y_jNvO.js} | 2 +- ...nHash-Df4j7HW3.js => IconHash-Bsu2htWI.js} | 2 +- ...> IconPlayerSkipForwardFilled-DO-WPkWS.js} | 2 +- ...obot-QtgNCzNZ.js => IconRobot-BQwymFKA.js} | 2 +- ..._ovR.js => IconSortDescending-nc0XizMI.js} | 2 +- ...CKIlrkvS.js => IconTrendingUp-DvwC6rkL.js} | 2 +- ...nUser-Bzi_KltT.js => IconUser-ectHbzBi.js} | 2 +- .../{Live-cR5MZGmO.js => Live-B7kSsFN1.js} | 2 +- ...ab-5PxRY34W.js => MessagesTab-Dcm_kMZs.js} | 2 +- .../{Modal-DllwOuRR.js => Modal-Da9xsK-A.js} | 2 +- ...rview-CzSNFKUS.js => Overview-DRVBiWsj.js} | 2 +- ...ab-1k1LKa_W.js => OverviewTab-BptDgpTp.js} | 2 +- ...ab-nC_K1Nmz.js => PlaybackTab-DX5SA_Uq.js} | 2 +- .../react/assets/ProjectDashboard-BAAQbkOV.js | 7 - .../react/assets/ProjectDashboard-DRHQbkve.js | 7 + ...p-XJ8IhR2k.js => ProviderChip-PiRoWNpI.js} | 2 +- .../{QATab-CMsiovWx.js => QATab-Dgj7TjI_.js} | 2 +- ...hTab-BwjAzqLj.js => SearchTab-BNg4qG1t.js} | 2 +- ...ab-CipJVVKQ.js => SessionsTab-1seW1KtH.js} | 2 +- ...tings-BStD5syP.js => Settings-Cnnpq-i1.js} | 2 +- ...agsTab-CbLDxWPD.js => TagsTab-CBR6F8Z3.js} | 2 +- ...o.js => TokenCompositionDonut-DsJdUSqI.js} | 2 +- ...b-BCso8Dz3.js => WorktreesTab-B6uk-r9G.js} | 2 +- ...ldTab-CA6f9UTO.js => YieldTab-Ck-altPN.js} | 2 +- ...-DQ-tTy3u.js => dashboardTabs-CJWioGLA.js} | 2 +- .../{index-DxsjU0Oz.js => index-DDMGE_ZF.js} | 20 +- .../static/react/assets/index-DL9ELn-S.css | 1 + .../static/react/assets/index-N1xvIAni.css | 1 - stackunderflow/static/react/index.html | 4 +- .../stackunderflow/cli/test_benchmark_cli.py | 114 ++ .../integration/test_route_perf_regression.py | 6 +- .../stackunderflow/reports/test_benchmark.py | 321 +++++ .../routes/test_benchmark_route.py | 142 +++ .../services/test_benchmark_stats.py | 182 +++ .../test_meta_agent_recommend_model.py | 52 + .../services/test_task_classifier.py | 116 ++ 73 files changed, 3667 insertions(+), 149 deletions(-) create mode 100644 docs/specs/benchmark-rubric-v1.md create mode 100644 stackunderflow-ui/src/components/dashboard/ModelWinsPanel.tsx create mode 100644 stackunderflow/reports/benchmark.py create mode 100644 stackunderflow/routes/benchmark.py create mode 100644 stackunderflow/services/benchmark_stats.py create mode 100644 stackunderflow/services/task_classifier.py rename stackunderflow/static/react/assets/{AgentsTab-ZL7pzav8.js => AgentsTab-DJ-0Trzo.js} (96%) rename stackunderflow/static/react/assets/{BookmarksTab-D5-cF5Yn.js => BookmarksTab-DAjJ72Pj.js} (96%) rename stackunderflow/static/react/assets/{BudgetsTab-CukAskCj.js => BudgetsTab-n3gfgUaU.js} (98%) rename stackunderflow/static/react/assets/{CodingHealthTab-CrhCZGhW.js => CodingHealthTab-Cz4e1hEF.js} (98%) rename stackunderflow/static/react/assets/{CommandsTab-CWrSh_N3.js => CommandsTab-D4oiPDTW.js} (93%) delete mode 100644 stackunderflow/static/react/assets/CompareTab-C7M_P6g3.js create mode 100644 stackunderflow/static/react/assets/CompareTab-CVqkwtRX.js rename stackunderflow/static/react/assets/{ContextReplayTab-BK8QfR7v.js => ContextReplayTab-DjTyR19Y.js} (98%) rename stackunderflow/static/react/assets/{CostTab-DmTsx3v9.js => CostTab-BAdmE1EC.js} (99%) rename stackunderflow/static/react/assets/{DataTable-DxCemy_F.js => DataTable-D0JcWuTA.js} (95%) rename stackunderflow/static/react/assets/{EstimatedCostMarker-BRbd4P-q.js => EstimatedCostMarker-DkSj8xDc.js} (90%) rename stackunderflow/static/react/assets/{FilterBar-CV18m0Rx.js => FilterBar-DuduzIn6.js} (98%) rename stackunderflow/static/react/assets/{ForksTab-D6x9mT4f.js => ForksTab-ovi_pavq.js} (96%) rename stackunderflow/static/react/assets/{IconActivity-DK0jJa88.js => IconActivity-BkaNd4nH.js} (86%) rename stackunderflow/static/react/assets/{IconAlertCircle-CqIiBOhp.js => IconAlertCircle-DyBnJSK0.js} (89%) rename stackunderflow/static/react/assets/{IconArrowRight-DHcdKZxB.js => IconArrowRight-xIaEMpRY.js} (89%) rename stackunderflow/static/react/assets/{IconArrowUp-kJ6mwirp.js => IconArrowUp-CszCUTPq.js} (94%) rename stackunderflow/static/react/assets/{IconCheck-DaUkhaCc.js => IconCheck-DwMBTi_m.js} (86%) rename stackunderflow/static/react/assets/{IconCircleX-Ci-XRc_E.js => IconCircleX-1-WYCT5P.js} (88%) rename stackunderflow/static/react/assets/{IconClock-CNOtVJvP.js => IconClock-Dl-xdEIm.js} (88%) rename stackunderflow/static/react/assets/{IconClockHour4-B8eFf-3y.js => IconClockHour4-UHU_zj__.js} (89%) rename stackunderflow/static/react/assets/{IconCode-B8NfAF9i.js => IconCode-DaQ_eMLh.js} (88%) rename stackunderflow/static/react/assets/{IconCopy-BZaMjQku.js => IconCopy-DA8lvnPB.js} (92%) rename stackunderflow/static/react/assets/{IconFileText-u1caIhwu.js => IconFileText-C9y_jNvO.js} (91%) rename stackunderflow/static/react/assets/{IconHash-Df4j7HW3.js => IconHash-Bsu2htWI.js} (89%) rename stackunderflow/static/react/assets/{IconPlayerSkipForwardFilled-DUER0OAK.js => IconPlayerSkipForwardFilled-DO-WPkWS.js} (95%) rename stackunderflow/static/react/assets/{IconRobot-QtgNCzNZ.js => IconRobot-BQwymFKA.js} (93%) rename stackunderflow/static/react/assets/{IconSortDescending-D85v_ovR.js => IconSortDescending-nc0XizMI.js} (90%) rename stackunderflow/static/react/assets/{IconTrendingUp-CKIlrkvS.js => IconTrendingUp-DvwC6rkL.js} (88%) rename stackunderflow/static/react/assets/{IconUser-Bzi_KltT.js => IconUser-ectHbzBi.js} (94%) rename stackunderflow/static/react/assets/{Live-cR5MZGmO.js => Live-B7kSsFN1.js} (97%) rename stackunderflow/static/react/assets/{MessagesTab-5PxRY34W.js => MessagesTab-Dcm_kMZs.js} (96%) rename stackunderflow/static/react/assets/{Modal-DllwOuRR.js => Modal-Da9xsK-A.js} (94%) rename stackunderflow/static/react/assets/{Overview-CzSNFKUS.js => Overview-DRVBiWsj.js} (98%) rename stackunderflow/static/react/assets/{OverviewTab-1k1LKa_W.js => OverviewTab-BptDgpTp.js} (98%) rename stackunderflow/static/react/assets/{PlaybackTab-nC_K1Nmz.js => PlaybackTab-DX5SA_Uq.js} (99%) delete mode 100644 stackunderflow/static/react/assets/ProjectDashboard-BAAQbkOV.js create mode 100644 stackunderflow/static/react/assets/ProjectDashboard-DRHQbkve.js rename stackunderflow/static/react/assets/{ProviderChip-XJ8IhR2k.js => ProviderChip-PiRoWNpI.js} (70%) rename stackunderflow/static/react/assets/{QATab-CMsiovWx.js => QATab-Dgj7TjI_.js} (97%) rename stackunderflow/static/react/assets/{SearchTab-BwjAzqLj.js => SearchTab-BNg4qG1t.js} (97%) rename stackunderflow/static/react/assets/{SessionsTab-CipJVVKQ.js => SessionsTab-1seW1KtH.js} (98%) rename stackunderflow/static/react/assets/{Settings-BStD5syP.js => Settings-Cnnpq-i1.js} (99%) rename stackunderflow/static/react/assets/{TagsTab-CbLDxWPD.js => TagsTab-CBR6F8Z3.js} (97%) rename stackunderflow/static/react/assets/{TokenCompositionDonut-BNnKqTro.js => TokenCompositionDonut-DsJdUSqI.js} (98%) rename stackunderflow/static/react/assets/{WorktreesTab-BCso8Dz3.js => WorktreesTab-B6uk-r9G.js} (98%) rename stackunderflow/static/react/assets/{YieldTab-CA6f9UTO.js => YieldTab-Ck-altPN.js} (96%) rename stackunderflow/static/react/assets/{dashboardTabs-DQ-tTy3u.js => dashboardTabs-CJWioGLA.js} (99%) rename stackunderflow/static/react/assets/{index-DxsjU0Oz.js => index-DDMGE_ZF.js} (51%) create mode 100644 stackunderflow/static/react/assets/index-DL9ELn-S.css delete mode 100644 stackunderflow/static/react/assets/index-N1xvIAni.css create mode 100644 tests/stackunderflow/cli/test_benchmark_cli.py create mode 100644 tests/stackunderflow/reports/test_benchmark.py create mode 100644 tests/stackunderflow/routes/test_benchmark_route.py create mode 100644 tests/stackunderflow/services/test_benchmark_stats.py create mode 100644 tests/stackunderflow/services/test_meta_agent_recommend_model.py create mode 100644 tests/stackunderflow/services/test_task_classifier.py diff --git a/docs/specs/benchmark-rubric-v1.md b/docs/specs/benchmark-rubric-v1.md new file mode 100644 index 00000000..36e6dc3c --- /dev/null +++ b/docs/specs/benchmark-rubric-v1.md @@ -0,0 +1,105 @@ +# Benchmark Rubric v1 — ratified defaults + +*Companion to `docs/specs/benchmark-engine.md` (issue #99). This file is the +ratification that spec §4.6 requires before the engine may ship: the engine +proposes, this document decides. The values here are the loaded defaults in +`reports/benchmark.py`; they are surfaced in every payload (`weights`, +`rubric_version`, `ci_level`, `success_threshold`) and overridable per call, so +tuning them is data, not a code change.* + +*"Rubric v1" is the version of **this scoring rubric**, not the product/PyPI +version. It has nothing to do with the maintainer-only product version and moves +on its own cadence.* + +--- + +## Composite weights + +The per-stratum composite is a weighted blend of three axes, each normalized +*within the stratum* so the comparison is like-for-like: + +| Axis | Weight | What it measures | +|---|---:|---| +| **success** | **0.45** | Did the work land — the tiered outcome signal (§ below). | +| **cost** | **0.35** | Dollars per successful outcome (min-max inverse; cheapest wins). | +| **effort** | **0.20** | Turns / retries (min-max inverse; fewest wins). | + +Weights sum to 1.0. Rationale: outcome dominates (a cheap failure is not a win), +cost is the flagship economic axis, effort is a real but noisier signal. + +**Wall-clock duration is descriptive only** — it includes human away-from-keyboard +time, so it is never scored into the winner. + +**Reasoning efficiency is descriptive only and is never scored.** Providers that +don't report a wire reasoning-token count read as 0, so cross-provider reasoning +is not apples-to-apples. It is surfaced (`reasoning_share`) for context, never +folded into the composite. + +## Success threshold τ + +``` +τ = 7.0 +``` + +A real LLM grade (`grades.success`, `session_quality_metrics`) at or above 7.0 +counts as a success on the grade tier; below counts as a failure. Fallback +(non-LLM) grades are never persisted, so only real grades reach this tier. + +## Success signal — tiered, highest-confidence-first + +Composed per session; the first tier with a signal decides `outcome_success`, +and the deciding tier is recorded. Sessions with no signal are `NULL` — excluded +from rates, but counted in the coverage figure shown beside every verdict. + +| Tier | Source | success = 1 | success = 0 | +|---|---|---|---| +| 1 ground truth | PR / CI via commit link | PR merged & not reverted, or CI passed | PR reverted, or CI failed | +| 2 code delta | `static_analysis_findings` | net-improved | net-regressed | +| 3 LLM grade | `grades.success` (real only) | ≥ τ | < τ | +| 4 behavioral | `session_mart` proxies | one-shot | high-retry (≥ 8 turns) | + +## Confidence level + +``` +CI_LEVEL = 0.90 +``` + +All intervals (Wilson for success rates; seeded percentile bootstrap for +cost/turns; difference effects) report at 90%. The maintainer may raise this to +0.95 per call; 0.90 is the default. + +## Statistical gates (honesty contract) + +Unchanged from spec §4 and pinned in `services/benchmark_stats.py`: + +| Gate | Value | +|---|---| +| Min sessions per model per stratum | 5 | +| Min qualifying models to compare a stratum | 2 | +| Min balanced sessions for a cross-task headline | 20 | +| Min relative cost effect | 10% | +| Min success effect | 10 percentage points | +| Min grade effect | 0.5 points | +| Multiple-comparison control | Benjamini–Hochberg FDR at α = 1 − CI_LEVEL | +| Bootstrap iterations / seed | 2000 / pinned (reproducible) | + +Below the sample floor the verdict is the literal string **"insufficient +evidence"** with a `confidence` of `none`. A cross-task winner must win the +composite in ≥ 2 strata (each a real, FDR-significant separation), never clearly +lose a stratum, and clear the balanced-sample floor. + +## Canonical intent taxonomy + +Ratified as the **6-label** set (adopting `ops` over the recommender's former +5), owned by `services/task_classifier.py` and shared by the tag service, the +mode recommender, and the benchmark: + +``` +build · fix · explore · refactor · test · ops +``` + +## Scope + +Phase 3 live-replay is **deferred** (no `benchmark run` in this MVP). The MVP is +the observational engine (spec §8 Moves 0–3): a natural experiment over the +user's own history, stated as such in every payload. diff --git a/stackunderflow-ui/src/components/dashboard/CompareTab.tsx b/stackunderflow-ui/src/components/dashboard/CompareTab.tsx index cf9680ba..f9abdadf 100644 --- a/stackunderflow-ui/src/components/dashboard/CompareTab.tsx +++ b/stackunderflow-ui/src/components/dashboard/CompareTab.tsx @@ -10,6 +10,7 @@ import { formatCost, formatNumber, formatModelName } from '../../services/format import { useCurrency } from '../../services/currency' import { shortenModelId } from '../../services/providerStyle' import { useFilters } from '../../services/filters' +import ModelWinsPanel from './ModelWinsPanel' // --------------------------------------------------------------------------- // CompareTab — v0.6.1 multi-provider polish. @@ -399,6 +400,9 @@ export default function CompareTab() { )} + + {/* "Which model wins" — outcome-aware benchmark beneath the cost table. */} + ) } diff --git a/stackunderflow-ui/src/components/dashboard/ModelWinsPanel.tsx b/stackunderflow-ui/src/components/dashboard/ModelWinsPanel.tsx new file mode 100644 index 00000000..5e4785da --- /dev/null +++ b/stackunderflow-ui/src/components/dashboard/ModelWinsPanel.tsx @@ -0,0 +1,296 @@ +import { useState } from 'react' +import { useQuery } from '@tanstack/react-query' +import { IconScale, IconAlertCircle } from '@tabler/icons-react' +import { getBenchmark, type BenchmarkPeriod } from '../../services/api' +import type { + BenchmarkReportData, + BenchmarkStratum, + BenchmarkModelRow, + BenchmarkConfidence, + BenchmarkCellVerdict, +} from '../../types/api' +import LoadingSpinner from '../common/LoadingSpinner' +import EmptyState from '../common/EmptyState' +import { formatCost } from '../../services/format' +import { useCurrency } from '../../services/currency' + +// --------------------------------------------------------------------------- +// ModelWinsPanel — "Which model wins" (spec 26 / issue #99). +// +// Renders `GET /api/benchmark` beneath the Compare tab's model×cost table: an +// observational benchmark over the user's own history. The point of the panel +// is honesty, made visual — every row carries n, coverage, a Wilson success CI, +// and a confidence chip; under-powered rows render greyed as "insufficient +// evidence" rather than a fake rank; and a method banner states the natural- +// experiment caveat up front. +// --------------------------------------------------------------------------- + +const PERIODS: { id: BenchmarkPeriod; label: string }[] = [ + { id: 'today', label: 'Today' }, + { id: 'week', label: '7d' }, + { id: 'month', label: '30d' }, + { id: 'all', label: 'All' }, +] + +function pct(x: number | null | undefined): string { + if (x === null || x === undefined || !Number.isFinite(x)) return '—' + return `${(x * 100).toFixed(0)}%` +} + +const CONFIDENCE_CLASS: Record = { + high: 'bg-green-500/10 text-green-600 dark:text-green-400', + medium: 'bg-blue-500/10 text-blue-600 dark:text-blue-400', + low: 'bg-yellow-500/10 text-yellow-700 dark:text-yellow-400', + none: 'bg-gray-500/10 text-gray-500 dark:text-gray-400', +} + +const CELL_VERDICT_CLASS: Record = { + clear: 'bg-green-500/10 text-green-600 dark:text-green-400', + weak: 'bg-yellow-500/10 text-yellow-700 dark:text-yellow-400', + 'insufficient evidence': 'bg-gray-500/10 text-gray-500 dark:text-gray-400', +} + +function Chip({ label, className }: { label: string; className: string }) { + return ( + + {label} + + ) +} + +function MethodBanner({ report }: { report: BenchmarkReportData }) { + const cov = report.coverage + return ( +
+ + + Based on {cov.sessions_total.toLocaleString()} sessions you already ran — + a natural experiment, not a controlled trial. Success measured on{' '} + {cov.sessions_scored.toLocaleString()}/{cov.sessions_total.toLocaleString()}{' '} + sessions · grade coverage {pct(cov.grade_coverage)}. Weights: success{' '} + {report.weights.success}, cost {report.weights.cost}, effort{' '} + {report.weights.effort} · {Math.round(report.ci_level * 100)}% CI. + +
+ ) +} + +function VerdictCard({ + report, + currency, +}: { + report: BenchmarkReportData + currency: ReturnType['currency'] +}) { + const v = report.verdict + if (!v.winning_model) { + return ( +
+
+ + + No cross-task winner yet + +
+ {v.caveats[0] && ( +

{v.caveats[0]}

+ )} +
+ ) + } + return ( +
+
+ + + {v.headline} + + +
+
+ {v.cost_per_outcome_usd !== null && ( + {formatCost(v.cost_per_outcome_usd, currency)} / successful outcome + )} + {v.runner_up && · runner-up {v.runner_up}} +
+
+ ) +} + +function ModelRow({ + row, + isWinner, + currency, +}: { + row: BenchmarkModelRow + isWinner: boolean + currency: ReturnType['currency'] +}) { + const wilson = row.success_rate.ci_wilson + const cpo = row.cost_per_outcome.point + const dim = row.qualified ? '' : 'opacity-50' + return ( + + + {isWinner && } + {row.model} + {!row.qualified && ( + insufficient evidence + )} + + + {row.n} + + + {pct(row.success_rate.point)} + {wilson && ( + + {' '} + [{pct(wilson[0])}–{pct(wilson[1])}] + + )} + + + {cpo !== null ? `${formatCost(cpo, currency)}/outcome` : '—'} + + + {pct(row.coverage)} + + + {row.composite.toFixed(2)} + + + ) +} + +function StratumCard({ + stratum, + currency, +}: { + stratum: BenchmarkStratum + currency: ReturnType['currency'] +}) { + return ( +
+
+ + {stratum.intent} × {stratum.size_band} + + +
+
+ + + + + + + + + + + + + {stratum.models.map(m => ( + + ))} + +
ModelnSuccess (90% CI)Cost / outcomeCoverageComposite
+
+
+ ) +} + +export default function ModelWinsPanel() { + const { currency } = useCurrency() + const [period, setPeriod] = useState('all') + + const { data, isLoading, error } = useQuery({ + queryKey: ['benchmark', period], + queryFn: () => getBenchmark(period), + staleTime: 60_000, + }) + + const report = data?.report + + return ( +
+
+
+ +

+ Which model wins +

+ + cost per successful outcome, per task type — from your own history + +
+
+ {PERIODS.map(p => ( + + ))} +
+
+ + {isLoading && } + + {error && ( +
+ Failed to load benchmark: {error instanceof Error ? error.message : 'Unknown error'} +
+ )} + + {!isLoading && !error && report && report.coverage.sessions_total === 0 && ( + } + title="Not enough history to compare models yet" + description="Once you've run a few sessions across more than one model, this panel compares them by task type — honestly." + /> + )} + + {!isLoading && !error && report && report.coverage.sessions_total > 0 && ( + <> + + + {report.strata.length > 0 && ( +
+

+ Per task type (intent × size) +

+ {report.strata.map(s => ( + + ))} +
+ )} + + )} +
+ ) +} diff --git a/stackunderflow-ui/src/services/api.ts b/stackunderflow-ui/src/services/api.ts index a1b4fcd1..3c85a806 100644 --- a/stackunderflow-ui/src/services/api.ts +++ b/stackunderflow-ui/src/services/api.ts @@ -17,6 +17,7 @@ import type { CostByProviderResponse, YieldResponse, ForksResponse, + BenchmarkResponse, PlanResponse, BudgetResponse, BudgetUpdate, @@ -453,6 +454,21 @@ export async function getForks(period: ForksPeriod = 'all'): Promise { + const params = new URLSearchParams({ period }) + return fetchJson(`${BASE}/benchmark?${params}`) +} + export async function getPlan(): Promise { return fetchJson(`${BASE}/plan`) } diff --git a/stackunderflow-ui/src/types/api.ts b/stackunderflow-ui/src/types/api.ts index 36b90d30..2821ca0d 100644 --- a/stackunderflow-ui/src/types/api.ts +++ b/stackunderflow-ui/src/types/api.ts @@ -677,6 +677,87 @@ export interface ForksResponse { warning: string } +// --------------------------------------------------------------------------- +// Comparative benchmark engine (`GET /api/benchmark`, spec 26 / issue #99). +// An observational "which model wins for your work" verdict over the user's own +// history, with the full statistical-honesty machinery. All cost fields are +// pre-converted to the active currency by the route. +// --------------------------------------------------------------------------- + +/** A cost point estimate with its confidence interval (currency-converted). */ +export interface BenchmarkCostBlock { + point: number | null + ci: [number, number] | null +} + +/** One model's row within a stratum. */ +export interface BenchmarkModelRow { + model: string + n: number + qualified: boolean + coverage: number + success_measured_n: number + success_rate: { point: number | null; ci_wilson: [number, number] | null } + cost_per_outcome: BenchmarkCostBlock + median_cost: BenchmarkCostBlock + median_turns: number + reasoning_share: number + composite: number +} + +export type BenchmarkCellVerdict = 'clear' | 'weak' | 'insufficient evidence' + +/** One (intent × size) stratum: like-for-like tasks. */ +export interface BenchmarkStratum { + intent: string + size_band: string + models: BenchmarkModelRow[] + assignment_balance: Record + cell_verdict: BenchmarkCellVerdict + winner: string | null + effect: { + success_risk_difference?: number + cost_relative_delta?: number + statistically_separated?: boolean + practically_separated?: boolean + } +} + +export type BenchmarkConfidence = 'none' | 'low' | 'medium' | 'high' + +export interface BenchmarkVerdict { + headline: string + winning_model: string | null + confidence: BenchmarkConfidence + cost_per_outcome_usd: number | null + runner_up: string | null + caveats: string[] +} + +export interface BenchmarkReportData { + verdict: BenchmarkVerdict + strata: BenchmarkStratum[] + coverage: { + sessions_total: number + sessions_scored: number + grade_coverage: number + } + rubric_version: number + weights: Record + ci_level: number + success_threshold: number + warning: string + method_notes: string[] +} + +export interface BenchmarkResponse { + period: string + scope: string + report: BenchmarkReportData + currency: CurrencyInfo + warning: string | null +} + /** * Plan + usage payload from `GET /api/plan`. Both fields are nullable — * when no plan is configured, the route returns `{plan: null, usage: null}` diff --git a/stackunderflow/cli.py b/stackunderflow/cli.py index 19295386..05156452 100644 --- a/stackunderflow/cli.py +++ b/stackunderflow/cli.py @@ -2268,6 +2268,223 @@ def memory_embed(batch): ) +# ── benchmark: outcome-aware "which model wins for your work" ───────────────── +# +# ``stackunderflow benchmark`` surfaces the comparative benchmark engine +# (spec 26 / issue #99): an observational, statistically-honest verdict over the +# user's own history — per-task-type winners, or an honest "insufficient +# evidence". ``show`` prints the leaderboard + per-stratum honesty; ``recommend`` +# picks a model for a described task. ``--json`` emits the same +# ``stackunderflow.memory/1`` envelope the ``memory`` namespace uses, so an agent +# can ask "which model for this refactor?" and splice a bounded, evidence- +# carrying answer straight into its context. There is deliberately no ``run`` +# subcommand — nothing is executed; the benchmark is computed from history. + +_BENCH_PERIOD_ALIASES = { + "today": "today", "week": "7days", "7days": "7days", + "month": "month", "30days": "30days", "all": "all", +} + + +def _bench_scope(period): + """Resolve a friendly period to a Scope, raising a Click error on a bad one.""" + from stackunderflow.reports.scope import parse_period + + spec = _BENCH_PERIOD_ALIASES.get(period) + if spec is None: + raise click.BadParameter( + f"Invalid period {period!r}. Valid: {', '.join(_BENCH_PERIOD_ALIASES)}", + param_hint="--period", + ) + return parse_period(spec) + + +def _bench_project_ids(conn, project): + """Resolve a ``--project`` slug/path to a project_ids list, or None for all.""" + if not project: + return None + slug = Path(project).name + try: + rows = conn.execute( + "SELECT id FROM projects WHERE slug = ?", (slug,) + ).fetchall() + except Exception: # noqa: BLE001 — advisory: a bad store scopes to nothing + return [] + return [int(r["id"]) for r in rows] + + +def _bench_pack(rows, budget): + """Greedily keep leading strata within ``budget`` estimated tokens. + + Returns ``(kept, truncated)``. A non-positive budget disables packing. + """ + from stackunderflow.cli_helpers import agent_output + + if not budget or budget <= 0: + return rows, False + kept: list = [] + for r in rows: + trial = [*kept, r] + if agent_output.estimate_tokens(trial) > budget and kept: + return kept, True + kept = trial + return kept, False + + +@cli.group("benchmark") +def benchmark_group(): + """Which model wins for the kind of work you actually do. + + An observational benchmark over your own history — a natural experiment you + already ran, not live replay. Every verdict carries n, coverage, confidence + intervals and a ``confidence`` label, and says "insufficient evidence" + rather than guess. Run any subcommand with ``--json`` for the stable, + token-bounded agent-output envelope. + """ + + +@benchmark_group.command("show") +@click.option("--period", default="all", show_default=True, help="today | week | month | all") +@click.option("--project", default=None, help="Project slug/path to scope to. Default: whole store.") +@click.option("--intent", default=None, help="Filter to one intent stratum (build/fix/explore/refactor/test/ops).") +@click.option("--context-budget", "context_budget", type=int, default=None, + help="Token budget for --json output (strata are packed to fit).") +@click.option("--json", "as_json", is_flag=True, default=False, help="Shortcut for --format json.") +@click.option("--format", "fmt", type=click.Choice(_VALID_FORMATS), default="text", show_default=True, + help="Output format. 'json' emits the stable agent-output envelope.") +def benchmark_show(period, project, intent, context_budget, as_json, fmt): + """Leaderboard + per-stratum honesty for the current scope.""" + json_mode = _memory_format(fmt, as_json) == "json" + budget = _resolve_context_budget(context_budget) + from stackunderflow.reports.benchmark import analyze_benchmark + + scope = _bench_scope(period) + conn = _open_store() + try: + project_ids = _bench_project_ids(conn, project) + report = analyze_benchmark( + conn, scope=scope, project_ids=project_ids, intent=intent, + ) + finally: + conn.close() + + if json_mode: + from stackunderflow.cli_helpers import agent_output + + rows, truncated = _bench_pack(report.get("strata") or [], budget) + envelope = agent_output.build_envelope( + command="benchmark", + query={"period": period, "project": project, "intent": intent}, + results=rows, + budget=budget, + truncated=truncated, + extra={ + "verdict": report.get("verdict"), + "coverage": report.get("coverage"), + "weights": report.get("weights"), + "rubric_version": report.get("rubric_version"), + "ci_level": report.get("ci_level"), + "warning": report.get("warning"), + }, + ) + click.echo(agent_output.render(envelope)) + return + + _emit_benchmark_text(report, period=period, scope=scope.label) + + +@benchmark_group.command("recommend") +@click.option("--intent", required=True, help="Task intent: build/fix/explore/refactor/test/ops.") +@click.option("--size", default=None, help="Task size band: tiny/small/med/large.") +@click.option("--language", default=None, help="Dominant language hint (e.g. python).") +@click.option("--project", default=None, help="Project slug/path to scope to.") +@click.option("--context-budget", "context_budget", type=int, default=None, help="Token budget for --json output.") +@click.option("--json", "as_json", is_flag=True, default=False, help="Shortcut for --format json.") +@click.option("--format", "fmt", type=click.Choice(_VALID_FORMATS), default="text", show_default=True, + help="Output format. 'json' emits the stable agent-output envelope.") +def benchmark_recommend(intent, size, language, project, context_budget, as_json, fmt): + """Outcome-aware model pick for a described task.""" + json_mode = _memory_format(fmt, as_json) == "json" + budget = _resolve_context_budget(context_budget) + from stackunderflow.reports.benchmark import recommend_from_history + + conn = _open_store() + try: + project_ids = _bench_project_ids(conn, project) + rec = recommend_from_history( + conn, intent=intent, size=size, language=language, project_ids=project_ids, + ) + finally: + conn.close() + + if json_mode: + from stackunderflow.cli_helpers import agent_output + + envelope = agent_output.build_envelope( + command="benchmark-recommend", + query={"intent": intent, "size": size, "language": language, "project": project}, + results=[rec], + budget=budget, + truncated=False, + ) + click.echo(agent_output.render(envelope)) + return + + click.echo(f"Task: intent={intent} size={size or 'any'} language={language or 'any'}") + if rec.get("recommended_model"): + click.echo( + f" → {rec['recommended_model']} " + f"(confidence: {rec.get('confidence')}, basis: {rec.get('basis')})" + ) + else: + click.echo(" → insufficient evidence") + if rec.get("rationale"): + click.echo(f" {rec['rationale']}") + + +def _emit_benchmark_text(report, *, period, scope): + """Human-readable leaderboard for ``benchmark show``.""" + v = report.get("verdict") or {} + cov = report.get("coverage") or {} + click.echo(f"Benchmark — {scope} (period: {period})") + click.echo("") + if v.get("winning_model"): + cpo = v.get("cost_per_outcome_usd") + cpo_s = f" at ${cpo:.4f}/successful outcome" if cpo is not None else "" + click.echo(f"Verdict: {v['headline']}{cpo_s}") + click.echo( + f" confidence: {v.get('confidence')} runner-up: {v.get('runner_up') or '—'}" + ) + else: + click.echo(f"Verdict: {v.get('headline', 'insufficient evidence')}") + for c in (v.get("caveats") or [])[:1]: + click.echo(f" {c}") + click.echo("") + click.echo( + f"Coverage: {cov.get('sessions_scored', 0)}/{cov.get('sessions_total', 0)} " + f"sessions scored · grade coverage {cov.get('grade_coverage', 0) * 100:.0f}%" + ) + strata = report.get("strata") or [] + if strata: + click.echo("") + click.echo("Per-stratum (intent × size):") + for s in strata: + head = f" {s['intent']} × {s['size_band']}: {s['cell_verdict']}" + if s.get("winner"): + head += f" — {s['winner']} leads" + click.echo(head) + for m in s.get("models", []): + sr = m["success_rate"]["point"] + sr_s = f"{sr * 100:.0f}%" if sr is not None else "n/a" + cpo = m["cost_per_outcome"]["point"] + cpo_s = f"${cpo:.4f}/outcome" if cpo is not None else "—" + floor = "" if m["qualified"] else " [below sample floor]" + click.echo( + f" {m['model']}: n={m['n']}, success {sr_s}, " + f"{cpo_s}, composite {m['composite']:.2f}{floor}" + ) + + # ── context replay: reconstruct what the model "saw" at a point in a session ── # # ``stackunderflow context-replay --at `` reconstructs the diff --git a/stackunderflow/reports/benchmark.py b/stackunderflow/reports/benchmark.py new file mode 100644 index 00000000..28d0938a --- /dev/null +++ b/stackunderflow/reports/benchmark.py @@ -0,0 +1,1033 @@ +"""Comparative benchmark engine — "which model wins for your work?" (issue #99). + +An **observational** benchmark over the user's own history: a natural +experiment they already ran, not live re-execution. Spec 26 makes the central +design call that this — not replay — is the credible, local-first, zero-cost, +always-available core. This module is the join + statistics layer over data +every other surface already computes per session; the hard part is not the +join, it is refusing to name a winner when the evidence can't support one. + +Design contract (mirrors :mod:`stackunderflow.reports.forks` / +:mod:`stackunderflow.reports.anomaly`): + +* **Advisory, never raises.** A schemaless store, an empty ``session_mart``, a + single-model store, or any arithmetic edge returns an empty-but-well-formed + verdict (``"insufficient evidence"``). Callers never wrap this for + correctness. +* **Read-only + scope-bounded.** All SQL is guarded by ``sqlite_master`` and + narrowed by the caller's :class:`Scope` + optional ``project_ids``. +* **Cost is a black box.** ``cost_usd`` is read **only** from ``session_mart`` + and never recomputed — keeps ``test_pricing_invariants`` green. +* **Honesty first (§4).** Sample floors, Wilson / bootstrap CIs, difference + effects, BH-FDR across the family, direct standardization (never pooled), and + "insufficient evidence" as a first-class verdict with a ``confidence`` label. + +The rubric weights + success threshold τ are **maintainer-owned** (ratified in +``docs/specs/benchmark-rubric-v1.md``); they are surfaced in the payload and +overridable via the ``weights`` argument — never silently hard-coded. +""" + +from __future__ import annotations + +import json +import sqlite3 +import statistics +from dataclasses import dataclass, field +from typing import Any + +from stackunderflow.reports.scope import Scope +from stackunderflow.services import benchmark_stats as bs +from stackunderflow.services import task_classifier + +__all__ = [ + "analyze_benchmark", + "RUBRIC_VERSION", + "DEFAULT_WEIGHTS", + "SUCCESS_THRESHOLD", + "recommend_from_history", +] + + +# ── rubric v1 (maintainer-owned; ratified in benchmark-rubric-v1.md) ───────── + +RUBRIC_VERSION = 1 +# Composite weights — surfaced in the payload and overridable. Sum to 1.0. +DEFAULT_WEIGHTS: dict[str, float] = {"success": 0.45, "cost": 0.35, "effort": 0.20} +# τ — grade-tier success threshold (a real LLM grade ≥ τ counts as success). +SUCCESS_THRESHOLD = 7.0 + +# Behavioural (Tier-4) proxy: a session with this many assistant turns is read +# as high-retry (a soft failure signal). One-shot sessions are a soft success. +_HIGH_RETRY_TURNS = 8 + +# The natural-experiment caveat, stated verbatim in every payload (§4.7). +NATURAL_EXPERIMENT_WARNING = ( + "This compares models over sessions you already ran — a natural " + "experiment, not a controlled trial. Models were not randomly assigned to " + "tasks, so the engine stratifies by task type and size and standardizes " + "across strata to control for the confounder it can measure (task " + "difficulty). It cannot control for the ones it can't (your skill drift " + "over time, per-project difficulty, prompt-quality differences)." +) + +_METHOD_NOTES: tuple[str, ...] = ( + "Observed history is a natural experiment, not a randomized trial.", + "Models are compared only within a stratum of comparable tasks (intent × " + "size); cross-task figures use direct standardization, never a pooled mean.", + "Success is composed from the highest-confidence signal available per " + "session (PR/CI → code-delta → LLM grade → behavioral); sessions with no " + "signal are excluded from rates but counted in coverage.", + "Tier-1 commit attribution is a coarse 24h + cwd heuristic — a signal, not " + "gospel.", + "Reasoning efficiency is descriptive only and is never scored into the " + "winner (providers that report 0 reasoning tokens aren't apples-to-apples).", + "A win must clear a practical effect floor and survive Benjamini–Hochberg " + "FDR control; below the sample floor the verdict is 'insufficient evidence'.", +) + + +# ── per-session fact ───────────────────────────────────────────────────────── + + +@dataclass(slots=True) +class _SessionFact: + session_id: str + project_id: int + primary_model: str + intent: str + size_band: str + language: str | None + cost_usd: float + num_turns: int + is_one_shot: bool + output_tokens: int + reasoning_tokens: int + first_ts: str + outcome_success: int | None # 1 / 0 / None (unmeasured) + outcome_tier: str | None # which tier decided it + + +# ── table guard ────────────────────────────────────────────────────────────── + + +def _table_exists(conn: sqlite3.Connection, name: str) -> bool: + """True when *name* is a queryable table or view (``messages`` is a view).""" + try: + row = conn.execute( + "SELECT 1 FROM sqlite_master " + "WHERE type IN ('table', 'view') AND name = ? LIMIT 1", + (name,), + ).fetchone() + except sqlite3.Error: + return False + return row is not None + + +# ── success-signal composition (tiered) ────────────────────────────────────── + + +def _outcome_from_ground_truth(outcomes: dict[str, Any]) -> int | None: + """Tier 1 — PR/CI ground truth. ``1`` merged&clean / CI pass, ``0`` reverted + / CI fail, ``None`` when neither is present.""" + prs = outcomes.get("prs") or [] + ci = outcomes.get("ci_runs") or [] + reverted = any(p.get("reverted_at") for p in prs) + merged_ok = any( + (p.get("state") == "merged") and not p.get("reverted_at") for p in prs + ) + ci_pass = any(c.get("status") == "success" for c in ci) + ci_fail = any(c.get("status") == "failure" for c in ci) + if reverted: + return 0 + if merged_ok or ci_pass: + return 1 + if ci_fail: + return 0 + return None + + +def _outcome_from_static(metric_summary: dict[str, Any]) -> int | None: + """Tier 2 — net code-delta. ``1`` net-improved, ``0`` net-regressed, + ``None`` when the session was analyzed but shows no net direction.""" + improved = sum(int(m.get("improved", 0) or 0) for m in metric_summary.values()) + regressed = sum(int(m.get("regressed", 0) or 0) for m in metric_summary.values()) + if improved > regressed: + return 1 + if regressed > improved: + return 0 + return None + + +def _outcome_from_grade(grade_success: float | None) -> int | None: + """Tier 3 — real LLM grade. ``1`` if ``grades.success ≥ τ`` else ``0``.""" + if grade_success is None: + return None + return 1 if grade_success >= SUCCESS_THRESHOLD else 0 + + +def _outcome_from_behavior(is_one_shot: bool, num_turns: int) -> int | None: + """Tier 4 — behavioral proxy. One-shot → ``1``; high-retry → ``0``; else + ``None`` (no confident behavioral read).""" + if is_one_shot: + return 1 + if num_turns >= _HIGH_RETRY_TURNS: + return 0 + return None + + +# ── data loading ───────────────────────────────────────────────────────────── + + +def _load_facts( + conn: sqlite3.Connection, + *, + scope: Scope | None, + project_ids: list[int] | None, +) -> list[_SessionFact]: + """Load one :class:`_SessionFact` per scoped session, or ``[]``. + + Cost/model/tokens come from ``session_mart`` (cost is never recomputed). + Intent + text-language are derived from the first user turn via the + canonical ``task_classifier``; size band from the session's real token + volume. Success is composed from the tiered signals in bulk-loaded side + tables. Every table touch is guarded so a partial store degrades to fewer + tiers rather than raising. + """ + if not _table_exists(conn, "session_mart") or not _table_exists(conn, "sessions"): + return [] + if project_ids is not None and len(project_ids) == 0: + return [] + + sql = ( + "SELECT sm.session_id AS session_id, sm.project_id AS project_id, " + " sm.primary_model AS primary_model, " + " COALESCE(sm.cost_usd, 0.0) AS cost_usd, sm.first_ts AS first_ts, " + " COALESCE(sm.input_tokens, 0) AS input_tokens, " + " COALESCE(sm.output_tokens, 0) AS output_tokens, " + " COALESCE(sm.assistant_message_count, 0) AS assistant_message_count, " + " COALESCE(sm.is_one_shot, 0) AS is_one_shot, " + " (SELECT m.content_text FROM messages m " + " WHERE m.session_fk = s.id AND m.role = 'user' " + " ORDER BY m.seq ASC LIMIT 1) AS first_user_text " + "FROM session_mart sm " + "JOIN sessions s ON s.session_id = sm.session_id " + "WHERE sm.primary_model IS NOT NULL AND sm.primary_model != '' " + ) + params: list[Any] = [] + if project_ids: + placeholders = ",".join("?" for _ in project_ids) + sql += f"AND sm.project_id IN ({placeholders}) " + params.extend(project_ids) + if scope is not None and scope.since is not None: + sql += "AND sm.first_ts >= ? " + params.append(scope.since) + if scope is not None and scope.until is not None: + sql += "AND sm.first_ts <= ? " + params.append(scope.until) + + try: + rows = conn.execute(sql, params).fetchall() + except sqlite3.Error: + return [] + if not rows: + return [] + + grades = _load_grades(conn) + static_lang, static_outcome = _load_static(conn, {r["session_id"] for r in rows}) + ground_truth = _load_ground_truth(conn, {r["session_id"] for r in rows}) + reasoning = _load_reasoning(conn, scope=scope, project_ids=project_ids) + + facts: list[_SessionFact] = [] + for r in rows: + sid = str(r["session_id"]) + text = str(r["first_user_text"] or "") + intent = task_classifier.classify_intent(text) + size_band = task_classifier.band_for_token_count( + int(r["input_tokens"] or 0) + int(r["output_tokens"] or 0) + ) + language = static_lang.get(sid) or task_classifier.dominant_language(text) + turns = int(r["assistant_message_count"] or 0) + one_shot = bool(r["is_one_shot"]) + + success, tier = _compose_success( + sid, + ground_truth=ground_truth, + static_outcome=static_outcome, + grade_success=grades.get(sid), + is_one_shot=one_shot, + num_turns=turns, + ) + rt, ot = reasoning.get(sid, (0, int(r["output_tokens"] or 0))) + facts.append( + _SessionFact( + session_id=sid, + project_id=int(r["project_id"] or 0), + primary_model=str(r["primary_model"]), + intent=intent, + size_band=size_band, + language=language, + cost_usd=float(r["cost_usd"] or 0.0), + num_turns=turns, + is_one_shot=one_shot, + output_tokens=int(ot or 0), + reasoning_tokens=int(rt or 0), + first_ts=str(r["first_ts"] or ""), + outcome_success=success, + outcome_tier=tier, + ) + ) + return facts + + +def _compose_success( + session_id: str, + *, + ground_truth: dict[str, dict[str, Any]], + static_outcome: dict[str, int | None], + grade_success: float | None, + is_one_shot: bool, + num_turns: int, +) -> tuple[int | None, str | None]: + """Walk the four tiers in precedence order; the first non-None wins.""" + gt = ground_truth.get(session_id) + if gt is not None: + val = _outcome_from_ground_truth(gt) + if val is not None: + return val, "ground_truth" + st = static_outcome.get(session_id) + if st is not None: + return st, "code_delta" + gr = _outcome_from_grade(grade_success) + if gr is not None: + return gr, "llm_grade" + bh = _outcome_from_behavior(is_one_shot, num_turns) + if bh is not None: + return bh, "behavioral" + return None, None + + +def _load_grades(conn: sqlite3.Connection) -> dict[str, float]: + """``session_id → grades.success`` from ``session_quality_metrics`` (real + grades only ever persist there).""" + if not _table_exists(conn, "session_quality_metrics"): + return {} + try: + rows = conn.execute( + "SELECT session_id, grades_json FROM session_quality_metrics" + ).fetchall() + except sqlite3.Error: + return {} + out: dict[str, float] = {} + for r in rows: + try: + grades = json.loads(r["grades_json"]) + if isinstance(grades, dict) and "success" in grades: + out[str(r["session_id"])] = float(grades["success"]) + except (TypeError, ValueError): + continue + return out + + +def _load_static( + conn: sqlite3.Connection, session_ids: set[str] +) -> tuple[dict[str, str], dict[str, int | None]]: + """Return ``(dominant_language, net_outcome)`` from ``static_analysis_findings``. + + Reuses the canonical ``get_session_quality`` reader per session that has + findings (bounded — most sessions have none), so the improved/regressed + polarity matches the rest of the product exactly. + """ + if not _table_exists(conn, "static_analysis_findings") or not session_ids: + return {}, {} + try: + rows = conn.execute( + "SELECT DISTINCT session_id FROM static_analysis_findings" + ).fetchall() + except sqlite3.Error: + return {}, {} + analyzed = {str(r["session_id"]) for r in rows} & session_ids + if not analyzed: + return {}, {} + + from stackunderflow.services import static_analysis + + langs: dict[str, str] = {} + outcomes: dict[str, int | None] = {} + for sid in analyzed: + try: + quality = static_analysis.get_session_quality(conn, sid) + except Exception: # noqa: BLE001,S112 — advisory: skip a bad session + continue + summary = quality.summary or {} + languages = summary.get("languages") or [] + if languages: + langs[sid] = str(languages[0]) + outcomes[sid] = _outcome_from_static(summary.get("metrics") or {}) + return langs, outcomes + + +def _load_ground_truth( + conn: sqlite3.Connection, session_ids: set[str] +) -> dict[str, dict[str, Any]]: + """Return ``session_id → outcomes`` for sessions with a commit link. + + Bounded to commit-linked sessions (most stores have few), reusing the + canonical ``outcome_attribution.get_outcomes_for_session`` so PR-matching + stays in one place. + """ + if not _table_exists(conn, "commit_session_link") or not session_ids: + return {} + try: + rows = conn.execute( + "SELECT DISTINCT session_id FROM commit_session_link" + ).fetchall() + except sqlite3.Error: + return {} + linked = {str(r["session_id"]) for r in rows} & session_ids + if not linked: + return {} + + from stackunderflow.services import outcome_attribution + + out: dict[str, dict[str, Any]] = {} + for sid in linked: + try: + out[sid] = outcome_attribution.get_outcomes_for_session(conn, sid) + except Exception: # noqa: BLE001,S112 — advisory: skip a bad session + continue + return out + + +def _load_reasoning( + conn: sqlite3.Connection, + *, + scope: Scope | None, + project_ids: list[int] | None, +) -> dict[str, tuple[int, int]]: + """``session_id → (reasoning_tokens, output_tokens)`` from ``usage_events``. + + Reasoning is descriptive-only (0 for providers with no wire count); this is + surfaced, never scored. ``reasoning_tokens`` stays a subset of output. + """ + if not _table_exists(conn, "usage_events"): + return {} + sql = ( + "SELECT session_id, " + " COALESCE(SUM(reasoning_tokens), 0) AS rt, " + " COALESCE(SUM(output_tokens), 0) AS ot " + "FROM usage_events WHERE 1=1 " + ) + params: list[Any] = [] + if project_ids: + placeholders = ",".join("?" for _ in project_ids) + sql += f"AND project_id IN ({placeholders}) " + params.extend(project_ids) + if scope is not None and scope.since is not None: + sql += "AND ts >= ? " + params.append(scope.since) + if scope is not None and scope.until is not None: + sql += "AND ts <= ? " + params.append(scope.until) + sql += "GROUP BY session_id " + try: + rows = conn.execute(sql, params).fetchall() + except sqlite3.Error: + return {} + return {str(r["session_id"]): (int(r["rt"] or 0), int(r["ot"] or 0)) for r in rows} + + +# ── per-model per-cell statistics ──────────────────────────────────────────── + + +@dataclass(slots=True) +class _ModelCell: + model: str + facts: list[_SessionFact] = field(default_factory=list) + + @property + def n(self) -> int: + return len(self.facts) + + @property + def qualified(self) -> bool: + return self.n >= bs.MIN_SESSIONS_PER_CELL + + def measured(self) -> list[_SessionFact]: + return [f for f in self.facts if f.outcome_success is not None] + + def success_count(self) -> int: + return sum(1 for f in self.measured() if f.outcome_success == 1) + + def success_rate(self) -> float | None: + m = self.measured() + return (self.success_count() / len(m)) if m else None + + def total_cost(self) -> float: + return sum(f.cost_usd for f in self.facts) + + def cost_per_outcome(self) -> float | None: + succ = self.success_count() + return (self.total_cost() / succ) if succ > 0 else None + + def median_cost(self) -> float: + return statistics.median([f.cost_usd for f in self.facts]) if self.facts else 0.0 + + def median_turns(self) -> float: + return statistics.median([f.num_turns for f in self.facts]) if self.facts else 0.0 + + def reasoning_share(self) -> float: + ot = sum(f.output_tokens for f in self.facts) + rt = sum(f.reasoning_tokens for f in self.facts) + return (rt / ot) if ot > 0 else 0.0 + + +def _cost_per_outcome_ci( + facts: list[_SessionFact], *, ci_level: float +) -> tuple[float, float] | None: + """Seeded ratio bootstrap of Σcost / Σsuccess. ``None`` when < 2 successes. + + Resamples sessions with replacement; each resample yields Σcost/Σsuccess + (resamples with no successes are skipped). Deterministic via the pinned + seed, so the CI is reproducible. + """ + pairs = [(f.cost_usd, 1 if f.outcome_success == 1 else 0) for f in facts] + total_succ = sum(s for _, s in pairs) + if total_succ < 2 or len(pairs) < 2: + return None + import random + + rng = random.Random(bs.SEED) # noqa: S311 — deterministic resampling, not crypto + n = len(pairs) + ratios: list[float] = [] + for _ in range(bs.BOOTSTRAP_ITERS): + cost_sum = 0.0 + succ_sum = 0 + for _ in range(n): + c, s = pairs[rng.randrange(n)] + cost_sum += c + succ_sum += s + if succ_sum > 0: + ratios.append(cost_sum / succ_sum) + if not ratios: + return None + ratios.sort() + alpha = (1.0 - ci_level) / 2.0 + return (bs.percentile(ratios, alpha), bs.percentile(ratios, 1.0 - alpha)) + + +def _two_proportion_pvalue(s1: int, n1: int, s2: int, n2: int) -> float: + """Two-sided two-proportion z-test p-value (normal approx, stdlib). + + Used only to feed Benjamini–Hochberg (§4.4). Degenerate inputs (zero + variance) return ``1.0`` — no evidence of a difference. + """ + if n1 == 0 or n2 == 0: + return 1.0 + p1, p2 = s1 / n1, s2 / n2 + p_pool = (s1 + s2) / (n1 + n2) + var = p_pool * (1.0 - p_pool) * (1.0 / n1 + 1.0 / n2) + if var <= 0: + return 1.0 + z = (p1 - p2) / (var ** 0.5) + return 2.0 * (1.0 - statistics.NormalDist().cdf(abs(z))) + + +# ── verdict assembly ───────────────────────────────────────────────────────── + + +def analyze_benchmark( + conn: sqlite3.Connection, + *, + scope: Scope | None = None, + project_ids: list[int] | None = None, + intent: str | None = None, + weights: dict[str, float] | None = None, + ci_level: float = bs.CI_LEVEL, +) -> dict[str, Any]: + """Compute the comparative benchmark verdict over *scope*. + + Args: + conn: Open store connection; reads only, guarded so a schemaless DB + returns an empty-but-valid verdict. + scope: Optional timestamp window (``None`` = all time). + project_ids: Optional ``projects.id`` filter (``None`` = whole store). + intent: Optional single-intent filter (only that stratum family). + weights: Composite weights; defaults to the ratified rubric v1. + ci_level: Confidence level for every interval (default 0.90). + + Returns: + The full report dict (see the module docstring / spec §6.1). Always + well-formed; ``verdict.headline`` is ``"insufficient evidence"`` on any + degenerate store. + """ + used_weights = _resolve_weights(weights) + try: + facts = _load_facts(conn, scope=scope, project_ids=project_ids) + except Exception: # noqa: BLE001 — advisory: never raise from the report + facts = [] + + if intent: + facts = [f for f in facts if f.intent == intent] + + return _assemble(facts, weights=used_weights, ci_level=ci_level) + + +def _resolve_weights(weights: dict[str, float] | None) -> dict[str, float]: + """Return normalized composite weights, defaulting to rubric v1.""" + if not weights: + return dict(DEFAULT_WEIGHTS) + picked = {k: float(weights.get(k, DEFAULT_WEIGHTS[k])) for k in DEFAULT_WEIGHTS} + total = sum(picked.values()) + if total <= 0: + return dict(DEFAULT_WEIGHTS) + return {k: v / total for k, v in picked.items()} + + +def _empty_report( + weights: dict[str, float], ci_level: float, *, sessions_total: int = 0 +) -> dict[str, Any]: + return { + "verdict": { + "headline": "insufficient evidence", + "winning_model": None, + "confidence": "none", + "cost_per_outcome_usd": None, + "runner_up": None, + "caveats": [ + "Not enough comparable evidence to name a winner yet.", + ], + }, + "strata": [], + "coverage": { + "sessions_total": sessions_total, + "sessions_scored": 0, + "grade_coverage": 0.0, + }, + "rubric_version": RUBRIC_VERSION, + "weights": weights, + "ci_level": ci_level, + "success_threshold": SUCCESS_THRESHOLD, + "warning": NATURAL_EXPERIMENT_WARNING, + "method_notes": list(_METHOD_NOTES), + } + + +def _assemble( + facts: list[_SessionFact], + *, + weights: dict[str, float], + ci_level: float, +) -> dict[str, Any]: + """Turn per-session facts into the stratified verdict payload.""" + if not facts: + return _empty_report(weights, ci_level) + + # ── coverage ────────────────────────────────────────────────────────── + sessions_total = len(facts) + sessions_scored = sum(1 for f in facts if f.outcome_success is not None) + grade_scored = sum(1 for f in facts if f.outcome_tier == "llm_grade") + coverage = { + "sessions_total": sessions_total, + "sessions_scored": sessions_scored, + "grade_coverage": round(grade_scored / sessions_total, 4) if sessions_total else 0.0, + } + + # ── stratify: (intent, size_band) → model → cell ────────────────────── + strata: dict[tuple[str, str], dict[str, _ModelCell]] = {} + for f in facts: + key = (f.intent, f.size_band) + cell = strata.setdefault(key, {}).setdefault( + f.primary_model, _ModelCell(model=f.primary_model) + ) + cell.facts.append(f) + + # ── p-value family for BH-FDR (success difference per cell-pair) ────── + pvalues: list[float] = [] + pval_index: dict[tuple[tuple[str, str], str, str], int] = {} + for key, models in strata.items(): + qualified = sorted(m for m, c in models.items() if c.qualified) + for i in range(len(qualified)): + for j in range(i + 1, len(qualified)): + a, b = models[qualified[i]], models[qualified[j]] + ma, mb = a.measured(), b.measured() + pval_index[(key, a.model, b.model)] = len(pvalues) + pvalues.append( + _two_proportion_pvalue( + a.success_count(), len(ma), b.success_count(), len(mb) + ) + ) + reject = bs.benjamini_hochberg(pvalues, alpha=1.0 - ci_level) if pvalues else [] + + def _pair_significant(key: tuple[str, str], m1: str, m2: str) -> bool: + idx = pval_index.get((key, m1, m2)) + if idx is None: + idx = pval_index.get((key, m2, m1)) + return bool(idx is not None and idx < len(reject) and reject[idx]) + + # ── per-stratum payload + cell winners ──────────────────────────────── + strata_payload: list[dict[str, Any]] = [] + # winner-tracking for the cross-task headline + clear_wins: dict[str, int] = {} + clear_losses: dict[str, int] = {} + balanced_n: dict[str, int] = {} + cell_win_widths: dict[str, list[float]] = {} + cost_accum: dict[str, tuple[float, int]] = {} # model → (Σcost, Σsucc) over clear-win cells + + for key in sorted(strata.keys()): + intent_lbl, size_lbl = key + models = strata[key] + qualified = [c for c in models.values() if c.qualified] + for c in qualified: + balanced_n[c.model] = balanced_n.get(c.model, 0) + c.n + + model_rows = [_model_row(c, ci_level=ci_level) for c in models.values()] + _fill_composites(model_rows, weights) + model_rows.sort(key=lambda r: (r["qualified"], r["composite"]), reverse=True) + + cell_verdict = "insufficient evidence" + winner: str | None = None + effect: dict[str, Any] = {} + qrows = [r for r in model_rows if r["qualified"]] + if len(qrows) >= bs.MIN_MODELS_PER_CELL: + top, second = qrows[0], qrows[1] + winner = top["model"] + sr_diff = bs.risk_difference( + top["success_rate"]["point"] or 0.0, + second["success_rate"]["point"] or 0.0, + ) + cost_rel = _cost_effect(top, second) + practical = ( + abs(sr_diff) >= bs.MIN_EFFECT_SUCCESS or cost_rel >= bs.MIN_EFFECT_COST + ) + statistical = _pair_significant(key, top["model"], second["model"]) + effect = { + "success_risk_difference": round(sr_diff, 4), + "cost_relative_delta": round(cost_rel, 4), + "statistically_separated": statistical, + "practically_separated": practical, + } + if practical and statistical: + cell_verdict = "clear" + clear_wins[winner] = clear_wins.get(winner, 0) + 1 + clear_losses[second["model"]] = clear_losses.get(second["model"], 0) + 1 + wc = models[winner] + cs, ss = cost_accum.get(winner, (0.0, 0)) + cost_accum[winner] = (cs + wc.total_cost(), ss + wc.success_count()) + w_ci = top["success_rate"].get("ci_wilson") or [0.0, 1.0] + cell_win_widths.setdefault(winner, []).append(w_ci[1] - w_ci[0]) + else: + cell_verdict = "weak" + + strata_payload.append( + { + "intent": intent_lbl, + "size_band": size_lbl, + "models": model_rows, + "assignment_balance": {c.model: c.n for c in models.values()}, + "cell_verdict": cell_verdict, + "winner": winner, + "effect": effect, + } + ) + + verdict = _headline( + intent_filter=None, + clear_wins=clear_wins, + clear_losses=clear_losses, + balanced_n=balanced_n, + cell_win_widths=cell_win_widths, + cost_accum=cost_accum, + ) + + return { + "verdict": verdict, + "strata": strata_payload, + "coverage": coverage, + "rubric_version": RUBRIC_VERSION, + "weights": weights, + "ci_level": ci_level, + "success_threshold": SUCCESS_THRESHOLD, + "warning": NATURAL_EXPERIMENT_WARNING, + "method_notes": list(_METHOD_NOTES), + } + + +def _model_row(cell: _ModelCell, *, ci_level: float) -> dict[str, Any]: + """Per-model row inside a stratum (matches spec §6.1).""" + sr = cell.success_rate() + measured = cell.measured() + wilson = ( + list(bs.wilson_interval(cell.success_count(), len(measured), ci_level=ci_level)) + if measured + else None + ) + cost_ci = bs.percentile_bootstrap_ci( + [f.cost_usd for f in cell.facts], statistic="median", ci_level=ci_level + ) + cpo = cell.cost_per_outcome() + cpo_ci = _cost_per_outcome_ci(cell.facts, ci_level=ci_level) + return { + "model": cell.model, + "n": cell.n, + "qualified": cell.qualified, + "coverage": round(len(measured) / cell.n, 4) if cell.n else 0.0, + "success_measured_n": len(measured), + "success_rate": { + "point": round(sr, 4) if sr is not None else None, + "ci_wilson": [round(x, 4) for x in wilson] if wilson else None, + }, + "cost_per_outcome": { + "point": round(cpo, 6) if cpo is not None else None, + "ci": [round(x, 6) for x in cpo_ci] if cpo_ci else None, + }, + "median_cost": { + "point": round(cell.median_cost(), 6), + "ci": [round(x, 6) for x in cost_ci], + }, + "median_turns": round(cell.median_turns(), 2), + "reasoning_share": round(cell.reasoning_share(), 4), + "composite": 0.0, # filled by _fill_composites (normalized across the cell) + } + + +def _fill_composites(rows: list[dict[str, Any]], weights: dict[str, float]) -> None: + """Fill each row's normalized composite in ``[0, 1]`` (mutates ``rows``). + + The composite blends three axes, each normalized **within the stratum** so + the comparison is like-for-like: success rate (already 0..1, higher wins), + cost (min-max inverse — cheapest gets 1.0), and effort/turns (min-max + inverse — fewest gets 1.0). Reasoning efficiency is deliberately absent — + descriptive only, never scored (§3.3). A single-model cell scores every + axis at its best (nothing to compare against). + """ + if not rows: + return + + def _cost_of(r: dict[str, Any]) -> float: + cpo = r["cost_per_outcome"]["point"] + return cpo if cpo is not None else r["median_cost"]["point"] + + costs = [_cost_of(r) for r in rows] + turns = [r["median_turns"] for r in rows] + for r in rows: + success = r["success_rate"]["point"] or 0.0 + cost_norm = _inverse_minmax(costs, _cost_of(r)) + effort_norm = _inverse_minmax(turns, r["median_turns"]) + composite = ( + weights["success"] * success + + weights["cost"] * cost_norm + + weights["effort"] * effort_norm + ) + r["composite"] = round(min(1.0, max(0.0, composite)), 4) + + +def _inverse_minmax(values: list[float], v: float) -> float: + """Min-max *inverse* normalization: the smallest value → 1.0, largest → 0.0. + + Used for cost + effort where lower is better. A flat set (all equal, or a + single model) → 1.0, so a lone model isn't penalized for having no rival. + """ + lo, hi = min(values), max(values) + if hi <= lo: + return 1.0 + return 1.0 - (v - lo) / (hi - lo) + + +def _cost_effect(top: dict[str, Any], second: dict[str, Any]) -> float: + """Relative cost advantage of the composite winner over the runner-up. + + Uses cost-per-outcome when both have it, else median cost. Positive ⇒ the + winner is cheaper by that fraction. + """ + tw = top["cost_per_outcome"]["point"] + sw = second["cost_per_outcome"]["point"] + if tw is not None and sw is not None: + return bs.relative_delta(tw, sw) + return bs.relative_delta( + top["median_cost"]["point"], second["median_cost"]["point"] + ) + + +def _headline( + *, + intent_filter: str | None, + clear_wins: dict[str, int], + clear_losses: dict[str, int], + balanced_n: dict[str, int], + cell_win_widths: dict[str, list[float]], + cost_accum: dict[str, tuple[float, int]], +) -> dict[str, Any]: + """Decide the cross-task winner (§4.4) — or refuse. + + A headline winner must win the composite in ≥2 strata (with a real, BH- + significant separation), never clearly lose a stratum, and clear the + balanced-sample floor. Otherwise the verdict is "insufficient evidence". + """ + candidates = [ + m + for m in clear_wins + if clear_wins[m] >= 2 + and clear_losses.get(m, 0) == 0 + and balanced_n.get(m, 0) >= bs.MIN_BALANCED_TOTAL + ] + if len(candidates) != 1: + return { + "headline": "insufficient evidence", + "winning_model": None, + "confidence": "none", + "cost_per_outcome_usd": None, + "runner_up": None, + "caveats": _headline_caveats(candidates, clear_wins, balanced_n), + } + + winner = candidates[0] + # runner-up = next model by clear wins (0 if none) + others = sorted( + (m for m in clear_wins if m != winner), + key=lambda m: clear_wins[m], + reverse=True, + ) + runner_up = others[0] if others else None + + cost_sum, succ_sum = cost_accum.get(winner, (0.0, 0)) + cost_per_outcome = (cost_sum / succ_sum) if succ_sum > 0 else None + + confidence = _confidence( + winner=winner, + clear_wins=clear_wins, + clear_losses=clear_losses, + balanced_n=balanced_n, + cell_win_widths=cell_win_widths, + ) + label = f"{winner} wins" + (f" for {intent_filter}" if intent_filter else "") + return { + "headline": label, + "winning_model": winner, + "confidence": confidence, + "cost_per_outcome_usd": round(cost_per_outcome, 6) if cost_per_outcome else None, + "runner_up": runner_up, + "caveats": [ + "Winner holds across " + f"{clear_wins[winner]} strata with no stratum where it clearly loses.", + NATURAL_EXPERIMENT_WARNING, + ], + } + + +def _headline_caveats( + candidates: list[str], clear_wins: dict[str, int], balanced_n: dict[str, int] +) -> list[str]: + if len(candidates) > 1: + return [ + "More than one model qualifies as a cross-task winner — no single " + "winner can be named. Compare the per-stratum table instead.", + ] + if clear_wins: + best = max(clear_wins, key=lambda m: clear_wins[m]) + return [ + f"The strongest model ({best}) wins {clear_wins[best]} stratum/strata " + f"(need ≥2) with {balanced_n.get(best, 0)} balanced sessions " + f"(need ≥{bs.MIN_BALANCED_TOTAL}) — not enough to headline a winner.", + ] + return ["Not enough comparable evidence to name a winner yet."] + + +def _confidence( + *, + winner: str, + clear_wins: dict[str, int], + clear_losses: dict[str, int], + balanced_n: dict[str, int], + cell_win_widths: dict[str, list[float]], +) -> str: + """Product-of-terms confidence (mirrors ``mode_recommender``) → bucket.""" + n = balanced_n.get(winner, 0) + sample_term = min(1.0, n / (2.0 * bs.MIN_BALANCED_TOTAL)) + wins = clear_wins.get(winner, 0) + agreement_term = min(1.0, wins / 3.0) # 3+ agreeing strata → full marks + widths = cell_win_widths.get(winner) or [1.0] + ci_term = max(0.0, 1.0 - (sum(widths) / len(widths))) + score = sample_term * agreement_term * ci_term + return bs.confidence_bucket(score) + + +# ── recommendation (outcome-aware; §1 / §6.2) ──────────────────────────────── + + +def recommend_from_history( + conn: sqlite3.Connection, + *, + intent: str, + size: str | None = None, + language: str | None = None, + scope: Scope | None = None, + project_ids: list[int] | None = None, + weights: dict[str, float] | None = None, + ci_level: float = bs.CI_LEVEL, +) -> dict[str, Any]: + """Outcome-aware model pick for a *described* task (the successor to the + cost-only ``mode_recommender``). + + Restricts the benchmark to the matching stratum family and returns the + winning model with its evidence — or an honest "insufficient evidence". + Always returns a well-formed dict; never raises. + """ + report = analyze_benchmark( + conn, + scope=scope, + project_ids=project_ids, + intent=intent, + weights=weights, + ci_level=ci_level, + ) + # Filter strata to the requested size (and language, when both known). + strata = report.get("strata") or [] + if size: + strata = [s for s in strata if s.get("size_band") == size] + + # Prefer a matching clear cell; fall back to the headline verdict. + best_cell = None + for s in strata: + if s.get("cell_verdict") == "clear" and s.get("winner"): + best_cell = s + break + + verdict = report.get("verdict") or {} + if best_cell is not None: + winner = best_cell["winner"] + row = next( + (m for m in best_cell["models"] if m["model"] == winner), None + ) + return { + "intent": intent, + "size": size, + "language": language, + "recommended_model": winner, + "confidence": "medium" if verdict.get("winning_model") != winner else verdict.get("confidence", "medium"), + "basis": "stratum", + "stratum": {"intent": best_cell["intent"], "size_band": best_cell["size_band"]}, + "evidence": row, + "rationale": ( + f"In {best_cell['intent']} × {best_cell['size_band']} tasks, " + f"{winner} wins on the composite with a real, significant " + f"separation from the runner-up." + ), + "rubric_version": RUBRIC_VERSION, + "weights": report.get("weights"), + } + + return { + "intent": intent, + "size": size, + "language": language, + "recommended_model": verdict.get("winning_model"), + "confidence": verdict.get("confidence", "none"), + "basis": "headline" if verdict.get("winning_model") else "insufficient_evidence", + "stratum": None, + "evidence": None, + "rationale": ( + verdict.get("caveats", ["Not enough comparable evidence yet."]) or [""] + )[0], + "rubric_version": RUBRIC_VERSION, + "weights": report.get("weights"), + } diff --git a/stackunderflow/routes/benchmark.py b/stackunderflow/routes/benchmark.py new file mode 100644 index 00000000..da6ba4d0 --- /dev/null +++ b/stackunderflow/routes/benchmark.py @@ -0,0 +1,243 @@ +"""``GET /api/benchmark`` — comparative "which model wins for your work" verdict. + +Thin HTTP wrapper around :func:`stackunderflow.reports.benchmark.analyze_benchmark`. +An observational benchmark over the user's own history (spec 26): per-task-type +verdicts with the full statistical-honesty machinery, or — just as often and +just as valuable — an honest "insufficient evidence". + +Mirrors ``routes/forks.py`` exactly: + +* **200 ms analytical tier** (the ``/api/yield`` / ``/api/optimize`` tier, not + the 100 ms mart tier) — it is a cross-session statistical composite, so it is + wrapped in the same read-through cache forks uses (keyed on store + scope + + project ids, self-invalidated by a sessions signature that moves on ingest). +* **Currency contract** — every dollar figure is pre-converted to the active + currency before send, applied to a deep copy outside the cache so an FX + change is picked up without recompute. +* A ``warning`` field carries the natural-experiment caveat inline. +""" + +from __future__ import annotations + +import copy +import threading +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, HTTPException, Query + +import stackunderflow.deps as deps +from stackunderflow.infra.currency import active_currency_payload +from stackunderflow.reports.benchmark import analyze_benchmark, recommend_from_history +from stackunderflow.reports.scope import parse_period +from stackunderflow.store import db + +router = APIRouter() + +# Read-through cache: the analysis walks every scoped session and re-derives +# intent per session, so it isn't free. Keyed on (store, scope, ids) plus a +# sessions signature (max last_ts, summed message_count) that any ingest bumps, +# so a stale entry can't outlive a refresh — the same contract forks/cost use. +# Currency conversion stays OUTSIDE the cache (applied to a deep copy). +_BENCH_CACHE: dict[ + tuple[str, str, tuple[int, ...] | None, str | None], tuple[tuple[str | None, int], dict] +] = {} +_BENCH_CACHE_LOCK = threading.Lock() + + +def _bench_signature(conn: Any, project_ids: list[int] | None) -> tuple[str | None, int]: + """(max last_ts, summed message_count) over the scoped sessions. + + ``project_ids is None`` = whole store. Any ingest that writes a message + bumps this signature and forces a recompute. Advisory: a bad store returns + a sentinel that simply misses the cache rather than raising. + """ + try: + if project_ids is None: + row = conn.execute( + "SELECT MAX(last_ts) AS max_ts, " + "COALESCE(SUM(message_count), 0) AS n FROM sessions" + ).fetchone() + elif not project_ids: + return (None, 0) + else: + placeholders = ",".join("?" for _ in project_ids) + row = conn.execute( + "SELECT MAX(last_ts) AS max_ts, " # noqa: S608 — placeholders are only ? marks + "COALESCE(SUM(message_count), 0) AS n " + f"FROM sessions WHERE project_id IN ({placeholders})", + tuple(project_ids), + ).fetchone() + except Exception: # noqa: BLE001 — advisory: a bad store just misses cache + return (None, -1) + if row is None: + return (None, 0) + return (row["max_ts"], int(row["n"] or 0)) + + +def _analyze_benchmark_cached( + conn: Any, *, scope: Any, project_ids: list[int] | None, intent: str | None +) -> dict: + """Read-through cache around :func:`analyze_benchmark` (returns USD report).""" + sig = _bench_signature(conn, project_ids) + key = ( + str(deps.store_path), + scope.label, + tuple(sorted(project_ids)) if project_ids is not None else None, + intent, + ) + with _BENCH_CACHE_LOCK: + cached = _BENCH_CACHE.get(key) + if cached is not None and cached[0] == sig: + return copy.deepcopy(cached[1]) + report = analyze_benchmark(conn, scope=scope, project_ids=project_ids, intent=intent) + with _BENCH_CACHE_LOCK: + _BENCH_CACHE[key] = (sig, report) + return copy.deepcopy(report) + + +# Friendly period superset — ``week`` maps to ``7days`` inside ``parse_period``. +# Mirrors forks / yield so all three beta surfaces accept the same selector. +_PERIOD_ALIASES = { + "today": "today", + "week": "7days", + "7days": "7days", + "month": "month", + "30days": "30days", + "all": "all", +} + +_PERIOD_QUERY = Query("all", description="today | week | month | all") +_LOG_PATH_QUERY = Query(None, description="Project log path; omit for whole-store") +_INTENT_QUERY = Query(None, description="Filter to one intent stratum (build/fix/…)") +_SIZE_QUERY = Query(None, description="Task size band (tiny/small/med/large)") +_LANG_QUERY = Query(None, description="Dominant language hint") + + +def _project_ids_for(conn: Any, path: str) -> list[int]: + """Resolve a log path to the ``projects.id`` list for its slug (own resolver).""" + slug = Path(path).name + try: + rows = conn.execute("SELECT id FROM projects WHERE slug = ?", (slug,)).fetchall() + except Exception: # noqa: BLE001 — advisory route, never 500 on a bad store + return [] + return [int(r["id"]) for r in rows] + + +def _convert_report_costs(report: dict, rate: float) -> None: + """Convert every dollar figure in the report to the active currency in place. + + Explicit walk (never a blanket multiply) so a schema change can't silently + double-convert a non-cost field. ``cost_usd`` originates in ``session_mart`` + and is only *displayed* in another currency — the invariant is untouched. + """ + if rate == 1.0: + return + verdict = report.get("verdict") or {} + if verdict.get("cost_per_outcome_usd") is not None: + verdict["cost_per_outcome_usd"] = float(verdict["cost_per_outcome_usd"]) * rate + for stratum in report.get("strata") or []: + for m in stratum.get("models") or []: + _convert_cost_block(m.get("cost_per_outcome"), rate) + _convert_cost_block(m.get("median_cost"), rate) + + +def _convert_cost_block(block: Any, rate: float) -> None: + """Scale a ``{"point": x, "ci": [lo, hi]}`` cost block by ``rate`` in place.""" + if not isinstance(block, dict): + return + if block.get("point") is not None: + block["point"] = float(block["point"]) * rate + ci = block.get("ci") + if isinstance(ci, list): + block["ci"] = [float(x) * rate for x in ci] + + +@router.get("/api/benchmark") +async def get_benchmark( + period: str = _PERIOD_QUERY, + log_path: str | None = _LOG_PATH_QUERY, + intent: str | None = _INTENT_QUERY, +): + """Return ``{period, scope, report, currency, warning}``. + + ``report`` is the full benchmark verdict with every dollar figure already + converted to the active currency. Scoped to a project when ``log_path`` (or + the active ``deps.current_log_path``) resolves to one, else the whole store. + """ + period = period if isinstance(period, str) else "all" + spec = _PERIOD_ALIASES.get(period) + if spec is None: + raise HTTPException( + status_code=400, + detail=f"Invalid period '{period}'. Valid: {', '.join(_PERIOD_ALIASES)}", + ) + scope = parse_period(spec) + + log_path_str = log_path if isinstance(log_path, str) else None + intent_str = intent if isinstance(intent, str) else None + path = log_path_str or deps.current_log_path + + conn = db.connect(deps.store_path) + try: + project_ids = _project_ids_for(conn, path) if path else None + report = _analyze_benchmark_cached( + conn, scope=scope, project_ids=project_ids, intent=intent_str + ) + finally: + conn.close() + + currency = active_currency_payload() + _convert_report_costs(report, currency["rate_from_usd"]) + + return { + "period": period, + "scope": scope.label, + "report": report, + "currency": currency, + "warning": report.get("warning"), + } + + +@router.get("/api/benchmark/recommend") +async def get_benchmark_recommend( + intent: str = Query(..., description="Task intent (build/fix/explore/refactor/test/ops)"), + size: str | None = _SIZE_QUERY, + language: str | None = _LANG_QUERY, + log_path: str | None = _LOG_PATH_QUERY, + period: str = _PERIOD_QUERY, +): + """Return the outcome-aware model recommendation for a described task.""" + period = period if isinstance(period, str) else "all" + spec = _PERIOD_ALIASES.get(period) + if spec is None: + raise HTTPException( + status_code=400, + detail=f"Invalid period '{period}'. Valid: {', '.join(_PERIOD_ALIASES)}", + ) + scope = parse_period(spec) + if not isinstance(intent, str) or not intent.strip(): + raise HTTPException(status_code=400, detail="intent is required") + + size_str = size if isinstance(size, str) else None + lang_str = language if isinstance(language, str) else None + log_path_str = log_path if isinstance(log_path, str) else None + path = log_path_str or deps.current_log_path + + conn = db.connect(deps.store_path) + try: + project_ids = _project_ids_for(conn, path) if path else None + rec = recommend_from_history( + conn, intent=intent, size=size_str, language=lang_str, + scope=scope, project_ids=project_ids, + ) + finally: + conn.close() + + currency = active_currency_payload() + rate = currency["rate_from_usd"] + if rate != 1.0 and isinstance(rec.get("evidence"), dict): + _convert_cost_block(rec["evidence"].get("cost_per_outcome"), rate) + _convert_cost_block(rec["evidence"].get("median_cost"), rate) + + return {"period": period, "scope": scope.label, "recommendation": rec, "currency": currency} diff --git a/stackunderflow/server.py b/stackunderflow/server.py index 8777a209..35f066c1 100644 --- a/stackunderflow/server.py +++ b/stackunderflow/server.py @@ -19,6 +19,7 @@ # Route modules from stackunderflow.routes import ( agent_teams, + benchmark, bookmarks, budgets, cfg, @@ -291,6 +292,7 @@ def _watcher_conn() -> "object": app.include_router(budgets.router) app.include_router(whatif.router) app.include_router(forks.router) +app.include_router(benchmark.router) app.include_router(patterns.router) app.include_router(worktrees.router) diff --git a/stackunderflow/services/benchmark_stats.py b/stackunderflow/services/benchmark_stats.py new file mode 100644 index 00000000..2a3c8121 --- /dev/null +++ b/stackunderflow/services/benchmark_stats.py @@ -0,0 +1,312 @@ +"""Small, pure, seeded statistics for the comparative benchmark engine. + +Spec 26 §4 ("statistical honesty — the crux"). A benchmark that always names a +winner is folklore with a progress bar; the whole value is refusing to conclude +when the evidence is thin, and separating a real difference from noise when it +is not. That discipline is entirely in this module. + +Everything here is: + +* **Pure + deterministic.** No store, no clock, no network. The bootstrap draws + from a ``random.Random(_SEED)`` seeded generator so two runs over the same + numbers are byte-identical (the report's read-through cache and the tests both + rely on this). +* **stdlib only.** ``statistics`` (incl. ``NormalDist`` for the z-score) — no + numpy, consistent with the rest of the tree. ``reports/anomaly.py`` already + sets this tone with a robust-MAD ``MIN_POINTS`` floor; the engine extends it. + +The functions, mapped to the spec: + +* :func:`wilson_interval` — §4.2 success-rate CI (correct for small n where the + normal approximation breaks). +* :func:`percentile_bootstrap_ci` — §4.2 cost/grade/turns CI (skewed + continuous), seeded. +* :func:`benjamini_hochberg` — §4.4 FDR control across the family of tests. +* :func:`standardized_difference` / :func:`pooled_rate` — §3.2/§4.7 direct + standardization vs the confounded pooled mean (Simpson's-paradox defense). +* sample floors + effect thresholds + :func:`confidence_bucket` — §4.1/§4.3/§4.5 + ("insufficient evidence" as a first-class verdict). +""" + +from __future__ import annotations + +import random +import statistics +from typing import Any + +__all__ = [ + "SEED", + "CI_LEVEL", + "BOOTSTRAP_ITERS", + "MIN_SESSIONS_PER_CELL", + "MIN_MODELS_PER_CELL", + "MIN_BALANCED_TOTAL", + "MIN_EFFECT_COST", + "MIN_EFFECT_SUCCESS", + "MIN_EFFECT_GRADE", + "z_for_confidence", + "wilson_interval", + "percentile", + "percentile_bootstrap_ci", + "benjamini_hochberg", + "pooled_rate", + "standardized_rate", + "standardized_difference", + "relative_delta", + "risk_difference", + "confidence_bucket", +] + + +# ── pinned tunables ────────────────────────────────────────────────────────── + +# Pinned so the seeded bootstrap is reproducible: two runs on the same store +# agree, and a test can assert byte-identical CIs. +SEED = 1729 + +# Default confidence level for every interval. The maintainer may set 0.95; +# 0.90 is the ratified default (docs/specs/benchmark-rubric-v1.md). +CI_LEVEL = 0.90 + +# Bootstrap resamples for a continuous-metric CI. 2000 is the spec's figure — +# enough for a stable 90% interval without making the live join expensive. +BOOTSTRAP_ITERS = 2000 + +# Sample-size floors — refuse before you mislead (§4.1). 5 mirrors +# ``anomaly.MIN_POINTS`` and ``mode_recommender``'s n/5 term. +MIN_SESSIONS_PER_CELL = 5 # a model needs ≥5 sessions in a stratum to be scored +MIN_MODELS_PER_CELL = 2 # need ≥2 qualifying models to compare a stratum at all +MIN_BALANCED_TOTAL = 20 # per model, across strata, for a headline verdict + +# Practical-effect floors — statistical separation is necessary, not sufficient +# (§4.3). A win must also clear a floor a human would care about. +MIN_EFFECT_COST = 0.10 # ≥10% relative cost difference +MIN_EFFECT_SUCCESS = 0.10 # ≥10 percentage points +MIN_EFFECT_GRADE = 0.5 # ≥0.5 grade points + + +# ── z-score ────────────────────────────────────────────────────────────────── + + +def z_for_confidence(ci_level: float = CI_LEVEL) -> float: + """Two-sided z critical value for ``ci_level`` (e.g. 0.90 → 1.6449). + + Uses ``statistics.NormalDist`` — stdlib, no scipy. ``ci_level`` is clamped + to a sane open interval so a degenerate 0 or 1 can't blow up ``inv_cdf``. + """ + ci = min(max(float(ci_level), 0.5), 0.999999) + return statistics.NormalDist().inv_cdf(1.0 - (1.0 - ci) / 2.0) + + +# ── Wilson score interval (proportions) ────────────────────────────────────── + + +def wilson_interval( + successes: int, n: int, *, ci_level: float = CI_LEVEL +) -> tuple[float, float]: + """Wilson score interval for a proportion — correct at small n. + + The normal approximation (p ± z·√(p(1-p)/n)) is badly wrong for small n or + p near 0/1 (it can leave [0,1]); Wilson is the standard fix and the reason + a 3-of-4 success rate reports honest uncertainty instead of a fake 0.75±. + + ``n == 0`` → ``(0.0, 1.0)`` (no information → widest possible interval); + callers gate on the sample floor long before this, but the function stays + total. The result is always clamped to ``[0, 1]``. + """ + if n <= 0: + return (0.0, 1.0) + z = z_for_confidence(ci_level) + p = successes / n + z2 = z * z + denom = 1.0 + z2 / n + center = (p + z2 / (2.0 * n)) / denom + margin = (z / denom) * ((p * (1.0 - p) / n + z2 / (4.0 * n * n)) ** 0.5) + lo = max(0.0, center - margin) + hi = min(1.0, center + margin) + return (lo, hi) + + +# ── percentile + bootstrap (continuous, skewed) ────────────────────────────── + + +def percentile(sorted_values: list[float], q: float) -> float: + """Linear-interpolation percentile of an already-sorted list. + + ``q`` in ``[0, 1]``. Matches numpy's default ``'linear'`` method so the + bootstrap CI edges line up with what a reader would compute by hand. + """ + if not sorted_values: + return 0.0 + if len(sorted_values) == 1: + return float(sorted_values[0]) + q = min(max(float(q), 0.0), 1.0) + rank = q * (len(sorted_values) - 1) + lo_idx = int(rank) + frac = rank - lo_idx + if lo_idx + 1 >= len(sorted_values): + return float(sorted_values[-1]) + lo = sorted_values[lo_idx] + hi = sorted_values[lo_idx + 1] + return float(lo + (hi - lo) * frac) + + +def percentile_bootstrap_ci( + values: list[float], + *, + statistic: str = "median", + iters: int = BOOTSTRAP_ITERS, + ci_level: float = CI_LEVEL, + seed: int = SEED, +) -> tuple[float, float]: + """Percentile bootstrap CI for a skewed continuous metric (cost / turns). + + Resamples ``values`` with replacement ``iters`` times, recomputes + ``statistic`` (``"median"`` or ``"mean"``) on each resample, and returns the + central ``ci_level`` percentile band of that bootstrap distribution. + + Deterministic: the resampling uses ``random.Random(seed)`` with ``seed`` + pinned to :data:`SEED`, so the same input yields byte-identical bounds on + every run. Empty → ``(0.0, 0.0)``; a single value → ``(v, v)`` (a + degenerate but honest interval — one point has no spread). + """ + clean = [float(v) for v in values] + if not clean: + return (0.0, 0.0) + if len(clean) == 1: + return (clean[0], clean[0]) + + stat_fn = statistics.mean if statistic == "mean" else statistics.median + # Deterministic pseudo-random resampling — reproducibility, not crypto. + rng = random.Random(seed) # noqa: S311 + n = len(clean) + draws: list[float] = [] + for _ in range(max(1, iters)): + sample = [clean[rng.randrange(n)] for _ in range(n)] + draws.append(float(stat_fn(sample))) + draws.sort() + alpha = (1.0 - ci_level) / 2.0 + return (percentile(draws, alpha), percentile(draws, 1.0 - alpha)) + + +# ── Benjamini–Hochberg FDR ─────────────────────────────────────────────────── + + +def benjamini_hochberg(pvalues: list[float], *, alpha: float = 0.10) -> list[bool]: + """Benjamini–Hochberg step-up FDR control. + + Testing M models × S strata × K metrics inflates false positives fast + (§4.4). BH bounds the expected false-discovery rate at ``alpha``: sort the + p-values ascending, find the largest rank ``k`` with ``p_(k) ≤ (k/m)·α``, + and reject **every** hypothesis up to that rank (the step-up — a hypothesis + can be rejected even if it fails its own threshold, because a smaller p + later in the order carries it). + + Returns a list of booleans aligned to the **input** order (``True`` = + reject / a real finding). Empty input → ``[]``. + """ + m = len(pvalues) + if m == 0: + return [] + order = sorted(range(m), key=lambda i: pvalues[i]) + max_k = 0 + for rank, idx in enumerate(order, start=1): + if pvalues[idx] <= (rank / m) * alpha: + max_k = rank + reject = [False] * m + for rank, idx in enumerate(order, start=1): + if rank <= max_k: + reject[idx] = True + return reject + + +# ── standardization vs pooling (Simpson's-paradox defense) ─────────────────── + + +def pooled_rate(cells: dict[Any, tuple[int, float]]) -> float: + """Naive pooled mean: ``Σ(n·rate) / Σn`` over a model's own strata. + + This is the *confounded* number — it re-imports the selection bias because + a model's own assignment mix (mostly-easy vs mostly-hard) weights it. Kept + for disclosure/contrast only; the engine never ranks on it. + """ + num = 0.0 + den = 0 + for n, rate in cells.values(): + num += n * rate + den += n + return (num / den) if den else 0.0 + + +def standardized_rate( + cells: dict[Any, tuple[int, float]], weights: dict[Any, float] +) -> float: + """Direct-standardized rate: a model's per-stratum rates under a *common* + stratum weighting, over strata present in ``weights``.""" + num = 0.0 + den = 0.0 + for stratum, w in weights.items(): + if stratum in cells and w > 0: + num += w * cells[stratum][1] + den += w + return (num / den) if den else 0.0 + + +def standardized_difference( + a_cells: dict[Any, tuple[int, float]], + b_cells: dict[Any, tuple[int, float]], +) -> float: + """Standardized ``rate(A) - rate(B)`` over strata where **both** have data. + + Direct standardization with a common weight per shared stratum (the + combined sample there). This is the honest comparison the spec mandates: + pooling A's and B's own means separately would let a lopsided assignment + (A drew the easy tasks, B the hard ones) manufacture or reverse a winner + — the textbook Simpson's paradox (§3.2, §4.7). ``0.0`` when no stratum is + shared (→ "untested here", never imputed). + """ + shared = set(a_cells) & set(b_cells) + weights: dict[Any, float] = { + s: float(a_cells[s][0] + b_cells[s][0]) for s in shared + } + if not weights: + return 0.0 + return standardized_rate(a_cells, weights) - standardized_rate(b_cells, weights) + + +# ── effect sizes ───────────────────────────────────────────────────────────── + + +def relative_delta(new: float, base: float) -> float: + """Signed relative change ``(base - new) / base``; positive ⇒ ``new`` lower. + + Used for the cost axis: a positive value means the candidate is cheaper by + that fraction. ``0.0`` when ``base`` is 0 (no meaningful ratio).""" + if base == 0: + return 0.0 + return (base - new) / base + + +def risk_difference(p_a: float, p_b: float) -> float: + """Risk difference ``p_a - p_b`` (percentage-point gap between two rates).""" + return p_a - p_b + + +# ── confidence label ───────────────────────────────────────────────────────── + + +def confidence_bucket(score: float) -> str: + """Map a ``[0, 1]`` confidence score to ``{none, low, medium, high}``. + + ``none`` is the "insufficient evidence" verdict — the single most-surfaced + field (§4.5). Thresholds are deliberately conservative: the engine should + reach ``high`` only when the sample, balance, CI width, and cross-stratum + agreement all line up. + """ + if score >= 0.66: + return "high" + if score >= 0.40: + return "medium" + if score >= 0.15: + return "low" + return "none" diff --git a/stackunderflow/services/meta_agent.py b/stackunderflow/services/meta_agent.py index a9f6edb2..55dc9da8 100644 --- a/stackunderflow/services/meta_agent.py +++ b/stackunderflow/services/meta_agent.py @@ -595,6 +595,49 @@ def to_dict(self) -> dict[str, Any]: }, }, }, + { + "type": "function", + "function": { + "name": "recommend_model_for_task", + "description": ( + "Recommend which model to use for a described task, based on " + "the user's own OUTCOMES — not just cost. Where " + "``recommend_mode`` ranks on cost alone, this consults the " + "comparative benchmark: for the matching task stratum (intent × " + "size) it returns the model that historically won on the " + "composite of success, cost-per-successful-outcome, and effort, " + "with its evidence. Returns 'insufficient_evidence' honestly " + "when the user's history can't support a call. Use for 'which " + "model should I use for this refactor?' style routing." + ), + "parameters": { + "type": "object", + "properties": { + "intent": { + "type": "string", + "description": ( + "Task intent. One of build / fix / explore / " + "refactor / test / ops. Required." + ), + "enum": ["build", "fix", "explore", "refactor", "test", "ops"], + }, + "size": { + "type": "string", + "description": ( + "Optional task size band: tiny / small / med / " + "large. Narrows to that stratum when given." + ), + "enum": ["tiny", "small", "med", "large"], + }, + "language": { + "type": "string", + "description": "Optional dominant language hint (e.g. python).", + }, + }, + "required": ["intent"], + }, + }, + }, ] @@ -1215,6 +1258,33 @@ def _exec_get_session_quality( return quality_to_dict(quality) +def _exec_recommend_model_for_task( + conn: sqlite3.Connection, args: dict[str, Any] +) -> dict[str, Any]: + """Outcome-aware model recommendation from the comparative benchmark. + + Hands off to :func:`reports.benchmark.recommend_from_history`, which is + advisory-never-raises and returns an honest ``insufficient_evidence`` basis + when the user's history can't support a call. ``intent`` is required. + """ + from stackunderflow.reports import benchmark + + intent = str(args.get("intent") or "").strip() + if not intent: + return {"error": "intent is required"} + valid = {"build", "fix", "explore", "refactor", "test", "ops"} + if intent not in valid: + return {"error": f"intent must be one of {sorted(valid)} (got {intent!r})"} + size = args.get("size") + language = args.get("language") + return benchmark.recommend_from_history( + conn, + intent=intent, + size=str(size) if size else None, + language=str(language) if language else None, + ) + + # Dispatcher table — name → (conn, args) callable. Keeping this flat # (instead of dynamic getattr) makes the surface explicit: a new tool # has to be added in three places: catalogue, dispatcher, and tests. @@ -1233,6 +1303,7 @@ def _exec_get_session_quality( "get_pr_outcomes": _exec_get_pr_outcomes, "get_ci_runs": _exec_get_ci_runs, "get_session_quality": _exec_get_session_quality, + "recommend_model_for_task": _exec_recommend_model_for_task, } diff --git a/stackunderflow/services/mode_recommender.py b/stackunderflow/services/mode_recommender.py index 8bcc65c5..8ac300b2 100644 --- a/stackunderflow/services/mode_recommender.py +++ b/stackunderflow/services/mode_recommender.py @@ -84,6 +84,8 @@ from datetime import UTC, datetime, timedelta from typing import Any +from stackunderflow.services import task_classifier + __all__ = [ "Recommendation", "extract_features", @@ -98,47 +100,11 @@ CACHE_TTL_HOURS = 24 -# Token bands for the "same shape" filter. Edges are token-count thresholds -# (chars/4 estimate) — anything below the upper bound of a band falls in it. -# Keeping bands wide on purpose: v1 is matching shape, not token-budget -# accounting. -TOKEN_BANDS: tuple[tuple[str, int], ...] = ( - ("tiny", 200), - ("small", 800), - ("med", 3000), - ("large", 10**9), # catch-all -) - -# Intent keyword lookup. Earlier patterns win on overlap (a prompt that -# says "fix the test" maps to ``fix``, not ``test``). Keep the keyword -# lists short and obvious — heuristic v1, not NLP. -_INTENT_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = ( - ( - "fix", - ("fix", "bug", "broken", "regression", "debug", "patch", - "error", "fail", "crash", "stack trace"), - ), - ( - "refactor", - ("refactor", "clean up", "rename", "extract", "rewrite", - "simplify", "deduplicate", "tidy", "untangle"), - ), - ( - "test", - ("test", "tests", "unit test", "pytest", "spec", "coverage", - "fixture", "snapshot test"), - ), - ( - "build", - ("build", "add", "implement", "create", "new feature", - "ship", "write a", "scaffold"), - ), - ( - "explore", - ("explain", "what does", "how does", "summarise", "summarize", - "describe", "trace", "show me", "find", "search"), - ), -) +# Token bands for the "same shape" filter — re-exported from the canonical +# ``task_classifier`` so the recommender, the tag service, and the benchmark +# share one definition (Move 0 of spec 26). Same thresholds this module has +# always used; ``_token_band`` below still applies them to a chars/4 estimate. +TOKEN_BANDS: tuple[tuple[str, int], ...] = task_classifier.TOKEN_BANDS # Language hints. Lowercased substring match. The list is intentionally # short and high-signal — extending it is cheap (no ALTER TABLE because @@ -211,23 +177,16 @@ def to_dict(self) -> dict[str, Any]: def _intent_of(prompt: str) -> str: - """Map the prompt to a coarse intent label. - - Scans the keyword lists in declaration order and returns the **first - label whose keyword set has the most hits**. Ties broken by - declaration order (so ``fix`` beats ``test`` on a "fix the test" - prompt). Default ``"explore"`` when nothing matches — read-only - questions are the most common no-keyword case. + """Map the prompt to a single coarse intent label. + + Delegates to the canonical :func:`task_classifier.classify_intent` (Move 0 + of spec 26) so the recommender, the tag service, and the benchmark agree on + the taxonomy. The canonical classifier adopts the 6-label set (adds + ``ops``) and resolves multi-intent prompts by a fixed precedence that + keeps this module's historical single-label picks intact (e.g. "fix the + failing test" → ``fix``). Default ``"explore"`` when nothing matches. """ - if not prompt: - return "explore" - lowered = prompt.lower() - best: tuple[str, int] = ("explore", 0) - for label, keywords in _INTENT_KEYWORDS: - hits = sum(1 for kw in keywords if kw in lowered) - if hits > best[1]: - best = (label, hits) - return best[0] + return task_classifier.classify_intent(prompt or "") def _token_band(prompt: str) -> str: diff --git a/stackunderflow/services/tag_service.py b/stackunderflow/services/tag_service.py index 49359c50..9112a839 100644 --- a/stackunderflow/services/tag_service.py +++ b/stackunderflow/services/tag_service.py @@ -11,6 +11,8 @@ from collections import defaultdict from pathlib import Path +from stackunderflow.services import task_classifier + logger = logging.getLogger(__name__) # Well-known GitHub language colors @@ -127,25 +129,11 @@ "intent:ops": "#64748b", # slate — deploy / config / infra } -# Intent detection patterns. Order matters: first match wins when a session -# has evidence of multiple intents, but we keep all matches (a session CAN -# have multiple intents — e.g. a "build" that ends in "fix"). -INTENT_PATTERNS = [ - # build — adding something new - (r"\b(add|adding|added|implement|implementing|implemented|create|creating|created|build|building|built|new feature|scaffold|scaffolding|set up|setup)\b", "intent:build"), - # fix — bug or error - (r"\b(fix|fixing|fixed|bug|bugs|broken|breaks|breaking|crash|crashes|crashing|error|errors|traceback|stack trace|exception|regression|doesn't work|not working|failing|failed)\b", "intent:fix"), - # explore — reading / understanding - (r"\b(explain|explaining|explained|understand|understanding|walk me through|how does|how do|what does|what is|where is|show me|why is|why does|read|reading|review|reviewing|reviewed|look at|trace)\b", "intent:explore"), - # refactor — restructuring without behavior change - (r"\b(refactor|refactoring|refactored|clean up|cleanup|cleaning up|simplify|simplifying|simplified|restructure|restructuring|reorganize|reorganizing|rename|renaming|extract|extracting|inline|consolidate|dedup|deduplicate)\b", "intent:refactor"), - # test — writing or running tests - (r"\b(test|tests|testing|tested|unit test|integration test|pytest|jest|vitest|mocha|jasmine|rspec|assert|asserts|asserting|mock|mocking|mocked|spec|specs|coverage|tdd)\b", "intent:test"), - # ops — deployment, config, infra - # NOTE: `.env` is matched as a separate alternative with lookarounds because - # word-boundary (\b) can't anchor a pattern starting with a non-word char (.). - (r"(?:\b(?:deploy|deploying|deployed|deployment|ci/cd|ci\b|cd\b|github actions|gitlab ci|jenkins|docker|dockerfile|kubernetes|k8s|terraform|ansible|helm|env var|environment variable|nginx|caddy|systemd|pm2)\b|(? set[str]: """Return the set of intent:* tags that match the given text. A session can legitimately have multiple intents (e.g. the user started - building something, hit an error, and debugged it). We return all matches. + building something, hit an error, and debugged it). We return all + matches. Delegates to the canonical multi-label classifier and + re-applies the ``intent:`` storage prefix, so tags, the recommender, + and the benchmark never drift on what a "fix" is. """ - matches: set[str] = set() if not combined_text: - return matches - for pattern, intent in INTENT_PATTERNS: - if re.search(pattern, combined_text, re.IGNORECASE): - matches.add(intent) - return matches + return set() + return { + f"intent:{label}" + for label in task_classifier.classify_intents(combined_text) + } def auto_tag_session(self, session_id: str, messages: list[dict]) -> list[str]: """Auto-detect tags for a session from its messages. diff --git a/stackunderflow/services/task_classifier.py b/stackunderflow/services/task_classifier.py new file mode 100644 index 00000000..f0d99335 --- /dev/null +++ b/stackunderflow/services/task_classifier.py @@ -0,0 +1,258 @@ +"""Canonical task classifier — one source of truth for intent / size / language. + +Move 0 of the comparative benchmark engine (issue #99, spec 26 §8). Before +this module three surfaces each classified a session's *task* their own way: + +* ``services/tag_service.py`` — regex patterns, **6** intent labels + (``build/fix/explore/refactor/test/ops``), multi-label (a session can carry + several intents), emitted as ``intent: