From 0e3dc816e76d9819b8d91655d010a236ebea5542 Mon Sep 17 00:00:00 2001 From: Abhishek Mishra Date: Sat, 20 Jun 2026 04:37:35 +0530 Subject: [PATCH 1/2] test: pre-release verification harness (make verify) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Standing release gate that turns 'did we miss anything?' into a number. Four layers: 1. wire coverage — fail if any endpoint lacks a wire test (static) 2. live read sweep — auto-enumerate + hit every no-arg read endpoint (live) 3. field-drop audit — fields the API returns but the SDK doesn't type (live) 4. helper coverage — every hand-written helper has a unit test (static) Live layers need SMALLEST_API_KEY; hard gates (1,2,4) fail the build, field-drop is a warning (tracked as spec completeness). Run: make verify. Also adds tests/custom/test_helpers_rest.py (KB/Audience/Campaign/Page — the helpers the harness flagged as untested). First run found: calls/agent-schema field-drops (escalated) + 4 untested helpers (now covered). mypy green (703 files). --- .fernignore | 4 + Makefile | 13 ++ scripts/verify.py | 221 ++++++++++++++++++++++++++++++ tests/custom/test_helpers_rest.py | 77 +++++++++++ 4 files changed, 315 insertions(+) create mode 100644 Makefile create mode 100644 scripts/verify.py create mode 100644 tests/custom/test_helpers_rest.py diff --git a/.fernignore b/.fernignore index 03e47068..87327a6a 100644 --- a/.fernignore +++ b/.fernignore @@ -37,3 +37,7 @@ RELEASE.md .fern/replay.lock .fern/replay.yml .gitattributes + +# Pre-release verification harness (hand-written) +scripts/** +Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..474b2ab7 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +.PHONY: verify test mypy + +# Pre-release verification harness — wire coverage + live read sweep + +# field-drop audit + helper coverage. Live layers need SMALLEST_API_KEY. +verify: + python scripts/verify.py + +# Static checks (mirror CI) +mypy: + mypy . + +test: + pytest -rP -n auto . diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 00000000..d53e4a23 --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Pre-release verification harness for smallestai. + +One command, four layers, one pass/fail coverage report. This is the standing +release gate: it turns "did we miss anything?" into a number. + + 1. WIRE COVERAGE (static) — every generated endpoint has a wire test + 2. LIVE READ SWEEP (live) — every no-arg read endpoint is reachable + parses + 3. FIELD-DROP AUDIT(live) — fields the API returns but the SDK doesn't type + (the spec-vs-reality class wire tests miss) + 4. HELPER COVERAGE (static) — every hand-written helper has a unit test + +Live layers need SMALLEST_API_KEY (+ optional SMALLEST_BASE_URL); skipped if absent. +Hard gates (1, 2, 4) fail the build; the field-drop audit (3) reports gaps as warnings +(tracked as spec work, not a build break). + + python scripts/verify.py # or: make verify +""" +import inspect +import os +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SRC = ROOT / "src" / "smallestai" +WIRE = ROOT / "tests" / "wire" +CUSTOM = ROOT / "tests" / "custom" + +GREEN, RED, YEL, DIM, END = "\033[32m", "\033[31m", "\033[33m", "\033[2m", "\033[0m" +failures = [] +warnings = [] + + +def hdr(t): + print(f"\n{'='*64}\n {t}\n{'='*64}") + + +# ---------------------------------------------------------------- layer 1 +def public_methods(client_py): + txt = client_py.read_text() + names = re.findall(r"^ def ([a-z][a-zA-Z0-9_]*)\(", txt, re.M) + return [n for n in names if n != "with_raw_response"] + + +def check_wire_coverage(): + hdr("1. WIRE COVERAGE — every endpoint has a wire test") + wire_files = list(WIRE.glob("test_*.py")) + + def wire_count_for(ns): + key = ns.replace("_", "") + for wf in wire_files: + if key in wf.stem.lower().replace("_", ""): + return len(re.findall(r"def test_", wf.read_text())) + return 0 + + tot_m = tot_w = 0 + for ns_dir in sorted((SRC / "atoms").glob("*/")): + cf = ns_dir / "client.py" + if not cf.exists(): + continue + ns = ns_dir.name + m = len(public_methods(cf)) + w = wire_count_for(ns) + tot_m += m + tot_w += w + if w < m: + failures.append(f"wire: {ns} has {w} wire tests for {m} methods") + print(f" {RED}FAIL{END} {ns:<26} methods={m} wiretests={w}") + else: + print(f" {GREEN}ok{END} {ns:<26} methods={m} wiretests={w}") + print(f" {DIM}totals: {tot_m} methods, {tot_w} wire tests{END}") + + +# ---------------------------------------------------------------- live client +def live_client(): + key = os.environ.get("SMALLEST_API_KEY") + if not key: + return None + from smallestai import SmallestAI + base = os.environ.get("SMALLEST_BASE_URL") + if base: + from smallestai.environment import SmallestAIEnvironment + base = base.rstrip("/") + ws = base.replace("https://", "wss://").replace("http://", "ws://") + env = SmallestAIEnvironment(atoms=f"{base}/atoms/v1", waves=base, waves_ws=ws) + return SmallestAI(api_key=key, environment=env) + return SmallestAI(api_key=key) + + +def read_methods(client): + """Yield (label, bound_method) for every no-required-arg read endpoint.""" + READ = re.compile(r"^(list|get_all|retrieve_all|search|get_acquired)") + EXTRA = {"get_user_details", "get_account_details"} + for ns_name in dir(client.atoms): + if ns_name.startswith("_"): + continue + ns = getattr(client.atoms, ns_name) + if "Client" not in type(ns).__name__: + continue + for m_name in dir(ns): + if m_name.startswith("_") or m_name == "with_raw_response": + continue + m = getattr(ns, m_name) + if not callable(m): + continue + if not (READ.match(m_name) or m_name in EXTRA): + continue + try: + req = [p for p in inspect.signature(m).parameters.values() + if p.default is p.empty and p.kind in (p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY)] + except (TypeError, ValueError): + req = [] + if req: + continue + yield f"{ns_name}.{m_name}", m + + +# ---------------------------------------------------------------- layer 2 +def check_live_sweep(client): + hdr("2. LIVE READ SWEEP — every no-arg read endpoint") + from smallestai.atoms.helpers import as_page + items_by_label = {} + n = ok = 0 + for label, method in read_methods(client): + n += 1 + try: + resp = method() + try: + pg = as_page(resp) + items_by_label[label] = pg.items + detail = f"{len(pg.items)} items" + except Exception: + detail = "ok (non-list)" + ok += 1 + print(f" {GREEN}PASS{END} {label:<46} {DIM}{detail}{END}") + except Exception as e: + failures.append(f"live: {label} -> {type(e).__name__}: {str(e)[:80]}") + print(f" {RED}FAIL{END} {label:<46} {type(e).__name__}: {str(e)[:60]}") + print(f" {DIM}{ok}/{n} read endpoints reachable{END}") + return items_by_label + + +# ---------------------------------------------------------------- layer 3 +def _extras(model, path, noise={"__v"}): + found = [] + if model is None: + return found + # aliases of declared fields: the unchecked-construct path can leave the raw alias + # key in model_extra even when the field is populated — not a real drop. + aliases = {getattr(f, "alias", None) + for f in (type(model).model_fields.values() if hasattr(type(model), "model_fields") else [])} + for k in (getattr(model, "model_extra", None) or {}): + if k not in noise and k not in aliases: + found.append(f"{path}.{k}") + for fname in (type(model).model_fields if hasattr(type(model), "model_fields") else {}): + v = getattr(model, fname, None) + if hasattr(v, "model_extra"): + found += _extras(v, f"{path}.{fname}") + elif isinstance(v, list) and v and hasattr(v[0], "model_extra"): + found += _extras(v[0], f"{path}.{fname}[0]") + return found + + +def check_field_drop(items_by_label): + hdr("3. FIELD-DROP AUDIT — API returns but SDK doesn't type (spec gaps)") + total = 0 + for label, items in sorted(items_by_label.items()): + if not items: + continue + drops = _extras(items[0], label.split(".")[0]) + if drops: + total += len(drops) + warnings.append(f"field-drop: {label} -> {len(drops)} untyped") + shown = ['.'.join(d.split('.')[1:]) for d in drops[:6]] + print(f" {YEL}GAP{END} {label:<46} {len(drops)} untyped: {DIM}{', '.join(shown)}{'…' if len(drops)>6 else ''}{END}") + else: + print(f" {GREEN}ok{END} {label:<46} all fields typed") + print(f" {DIM}{total} untyped fields total (tracked as 5.1.x spec completeness){END}") + + +# ---------------------------------------------------------------- layer 4 +def check_helper_coverage(): + hdr("4. HELPER COVERAGE — every hand-written helper has a test") + init = (SRC / "atoms" / "helpers" / "__init__.py").read_text() + names = re.findall(r"\"([A-Za-z_]+)\"", init.split("__all__")[-1]) if "__all__" in init else [] + test_blob = "\n".join(p.read_text() for p in CUSTOM.glob("test_*.py")) + for name in names: + if re.search(rf"\b{name}\b", test_blob): + print(f" {GREEN}ok{END} {name}") + else: + failures.append(f"helper: {name} has no test in tests/custom") + print(f" {RED}FAIL{END} {name} — no test references it") + + +# ---------------------------------------------------------------- main +def main(): + print(f"{DIM}smallestai pre-release verification — {ROOT.name}{END}") + check_wire_coverage() + client = live_client() + if client is None: + hdr("2-3. LIVE LAYERS — SKIPPED (set SMALLEST_API_KEY to run)") + warnings.append("live layers skipped (no SMALLEST_API_KEY)") + else: + items = check_live_sweep(client) + check_field_drop(items) + check_helper_coverage() + + hdr("REPORT") + for w in warnings: + print(f" {YEL}WARN{END} {w}") + if failures: + for f in failures: + print(f" {RED}FAIL{END} {f}") + print(f"\n {RED}VERIFY FAILED — {len(failures)} hard failure(s){END}") + sys.exit(1) + print(f"\n {GREEN}VERIFY PASSED{END} ({len(warnings)} warning(s), 0 hard failures)") + + +if __name__ == "__main__": + main() diff --git a/tests/custom/test_helpers_rest.py b/tests/custom/test_helpers_rest.py new file mode 100644 index 00000000..5e709b39 --- /dev/null +++ b/tests/custom/test_helpers_rest.py @@ -0,0 +1,77 @@ +"""Unit tests for the hand-written REST helpers (KB, Audience, Campaign) + Page. + +No network: we monkeypatch the `requests` module each helper imports and assert it +builds the right URL with bearer auth. Guards the helpers the verify harness flagged +as untested. +""" +import smallestai.atoms.helpers.audience as audience_mod +import smallestai.atoms.helpers.campaign as campaign_mod +import smallestai.atoms.helpers.kb as kb_mod +from smallestai.atoms.helpers import KB, Audience, Campaign, Page + + +class _FakeResp: + def __init__(self): + self.calls = [] + + def raise_for_status(self): + return None + + def json(self): + return {"status": True, "data": []} + + +class _FakeRequests: + """Records (method, url, headers) and returns a canned response.""" + + def __init__(self): + self.last = None + self._resp = _FakeResp() + + def _record(self, method): + def fn(url, headers=None, **kw): + self.last = (method, url, headers or {}) + return self._resp + return fn + + def __getattr__(self, name): + if name in ("get", "post", "put", "patch", "delete"): + return self._record(name) + raise AttributeError(name) + + +def _patch(monkeypatch, module): + fake = _FakeRequests() + monkeypatch.setattr(module, "requests", fake) + return fake + + +def test_page_defaults(): + p = Page() + assert p.items == [] and p.total_count is None and p.has_more is None + p2 = Page(items=[1, 2], total_count=2) + assert p2.items == [1, 2] and p2.total_count == 2 + + +def test_kb_list_builds_url_with_auth(monkeypatch): + fake = _patch(monkeypatch, kb_mod) + KB(base_url="https://api.example/atoms/v1", api_key="sk_test").list() + method, url, headers = fake.last + assert method == "get" and url.startswith("https://api.example/atoms/v1") + assert headers.get("Authorization") == "Bearer sk_test" + + +def test_audience_list_builds_url_with_auth(monkeypatch): + fake = _patch(monkeypatch, audience_mod) + Audience(base_url="https://api.example/atoms/v1", api_key="sk_test").list() + method, url, headers = fake.last + assert method == "get" and url.startswith("https://api.example/atoms/v1") + assert headers.get("Authorization") == "Bearer sk_test" + + +def test_campaign_list_builds_url_with_auth(monkeypatch): + fake = _patch(monkeypatch, campaign_mod) + Campaign(base_url="https://api.example/atoms/v1", api_key="sk_test").list() + method, url, headers = fake.last + assert method == "get" and url.startswith("https://api.example/atoms/v1") + assert headers.get("Authorization") == "Bearer sk_test" From 2b171db508471c0427b4a6823393a3e2fa65b4d0 Mon Sep 17 00:00:00 2001 From: Abhishek Mishra Date: Sat, 20 Jun 2026 04:41:39 +0530 Subject: [PATCH 2/2] ci: run verify.py static gates (wire + helper coverage) in compile job Makes the verification harness self-enforcing: a new endpoint without a wire test, or a hand-written helper without a unit test, now fails CI automatically (no need to remember 'make verify'). Live layers skip without SMALLEST_API_KEY. --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c25ec6c9..90d34127 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,10 @@ jobs: run: poetry install - name: Compile run: poetry run mypy . + - name: Verify (static gates — wire coverage + helper coverage) + # Live layers (read sweep + field-drop) auto-skip without SMALLEST_API_KEY. + # Fails the build if a new endpoint has no wire test, or a helper has no test. + run: poetry run python scripts/verify.py test: runs-on: ubuntu-latest steps: