week_8b: Module C (The Librarian) — docs, final metrics, and the regression gate

.github/workflows/librarian_regression.yml

@@ -0,0 +1,67 @@
+name: Librarian Regression Gate
+# Module C's accuracy gate. The unit suite proves the code runs; this proves the
+# pipeline still *decides* correctly over the golden set.
+#
+# Deliberately hermetic: no DB, no API key, no model download. The semantic
+# reports (C.1 recall, C.2 top-1, the C.3 ECE gate, C.4 decision accuracy) need
+# live CRE vectors and are measured locally — seeding a candidate pool from
+# golden text offline is the leakage the hub firewall exists to strip, so a CI
+# job that "measured" them would be measuring nothing.
+#
+# What this gate does catch is the failure that actually bit us: a Module B
+# schema change silently rejecting every row at the C.0 boundary, which used to
+# leave the explicit gate with nothing to count and still exit 0.
+
+on:
+  pull_request:
+    paths:
+      - 'application/utils/librarian/**'
+      - 'application/tests/librarian/**'
+      - 'scripts/evaluate_librarian.py'
+      - '.github/workflows/librarian_regression.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'application/utils/librarian/**'
+      - 'application/tests/librarian/**'
+      - 'scripts/evaluate_librarian.py'
+
+permissions:
+  contents: read
+
+jobs:
+  regression:
+    name: Librarian Regression Gate
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          # This job only reads the tree; leaving the token in .git/config would
+          # expose it to anything the test run executes.
+          persist-credentials: false
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12.3'
+
+      - name: Install python dependencies
+        # Installed straight into the runner's interpreter rather than through
+        # `make install-python`: that target builds a venv (needing an apt
+        # `virtualenv` the other workflows install first) and then runs
+        # `playwright install`, neither of which a hermetic librarian run uses.
+        # The steps below call `python` directly, so a venv the job never
+        # activates would leave them running against a bare interpreter.
+        run: |
+          pip install --upgrade pip setuptools
+          pip install -r requirements-dev.txt
+
+      - name: Librarian unit suite
+        run: python -m unittest discover -s application/tests/librarian -p '*_test.py' -t .
+
+      - name: Golden-set decision gate
+        # Non-zero on: a failed explicit-slice gate (C.0.5 must be 100%), or a
+        # C.0 boundary that rejected the whole dataset.
+        run: |
+          python scripts/evaluate_librarian.py \
+            --dataset application/tests/librarian/fixtures/golden_dataset.json

.gitignore

@@ -69,6 +69,8 @@ standards_cache.sqlite
 !AGENTS.md
 !docs/faq.md
 !docs/Mid_eval_blog_gsoc2026/module_B_mideval_blog.md
+!application/utils/librarian/README.md
+!docs/gsoc_2026_module_c/*.md
 
 ### Dev DBDumps
 *.sql

application/cmd/cre_main.py

@@ -1032,11 +1032,23 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
     if args.upstream_sync:
         download_graph_from_upstream(args.cache_file)
     if args.run_librarian or args.librarian_dry_run:
-        run_librarian(
-            cache_file=args.cache_file,
-            dry_run=args.librarian_dry_run or not args.run_librarian,
-            source_jsonl=args.librarian_source,
-        )
+        # --run_id selects the live path: drain Module B's knowledge_queue for
+        # that run. Without it, the fixture walk-through stays the default so
+        # the pre-W8 command keeps behaving the way it always has.
+        run_id = (getattr(args, "run_id", "") or "").strip()
+        if run_id:
+            run_librarian_live(
+                cache_file=args.cache_file,
+                pipeline_run_id=run_id,
+                dry_run=args.librarian_dry_run,
+                envelopes_out=args.librarian_envelopes_out,
+            )
+        else:
+            run_librarian(
+                cache_file=args.cache_file,
+                dry_run=args.librarian_dry_run or not args.run_librarian,
+                source_jsonl=args.librarian_source,
+            )
 
 
 def ai_client_init(database: db.Node_collection):
@@ -1238,6 +1250,67 @@ def run_librarian(
     )
 
 
+def run_librarian_live(
+    cache_file: str,
+    pipeline_run_id: str,
+    dry_run: bool = False,
+    envelopes_out: Optional[str] = None,
+) -> None:
+    """Module C entrypoint against Module B's live queue (W8).
+
+    The counterpart to ``--run_noise_filter``: where ``run_librarian`` walks a
+    JSONL fixture and logs shortlists, this drains the ``knowledge_queue`` rows
+    Module B wrote for ``pipeline_run_id`` through C.0->C.4 and stamps
+    ``consumed_at`` on the ones it finished. Prints the ``RunSummary`` as JSON
+    on stdout so the OIE orchestrator can read it, exactly as Module B does.
+
+    Still writes no links: the graph/review writers are W8b. What a real run
+    does write is the envelopes (as JSONL, to ``envelopes_out``) and one column
+    on B's queue. Those two go together — retiring a row whose envelope was
+    discarded would lose the chunk — so a non-dry run requires an output path.
+    ``dry_run`` writes neither.
+
+    Ops note: unchanged from ``run_librarian`` — opt-in CLI only, not on the
+    Procfile, not wired into the web app or the worker. It calls the paid
+    embedding API, so a deployment never triggers it on its own.
+    """
+    from datetime import datetime, timezone
+
+    from application.utils.librarian.config_loader import load_config
+    from application.utils.librarian.envelope_sink import (
+        JsonlEnvelopeSink,
+        NullEnvelopeSink,
+    )
+    from application.utils.librarian.factory import build_components
+    from application.utils.librarian.queue_runner import run_librarian_queue
+
+    sink = JsonlEnvelopeSink(envelopes_out) if envelopes_out else NullEnvelopeSink()
+    if not dry_run and envelopes_out is None:
+        raise SystemExit(
+            "--run_librarian --run_id needs --librarian_envelopes_out <path>: a "
+            "real run marks queue rows consumed, so the envelopes it built have "
+            "to land somewhere first. Add --librarian_dry_run to run without "
+            "writing anything."
+        )
+
+    cfg = load_config()
+    database = db_connect(path=cache_file)
+    components = build_components(database, config=cfg)
+
+    # The CLI boundary is the one place a clock read belongs; everything below
+    # takes `at` as an argument so a run stays reproducible.
+    summary = run_librarian_queue(
+        database.session,
+        pipeline_run_id,
+        components,
+        cfg,
+        at=datetime.now(timezone.utc),
+        sink=sink,
+        dry_run=dry_run,
+    )
+    print(summary.to_json())
+
+
 def regenerate_embeddings(db_url: str) -> None:
     """Wipe all embedding rows, then rebuild (CRE + every node type) like ``--generate_embeddings``."""
     from application.prompt_client import prompt_client as prompt_client

application/tests/librarian/config_loader_test.py

@@ -16,6 +16,9 @@ def test_defaults_when_env_unset(self):
         self.assertEqual(cfg.top_k_retrieval, 20)
         self.assertEqual(cfg.top_k_rerank, 5)
         self.assertEqual(cfg.link_threshold, 0.8)
+        # 1.0 is the identity transform: an honestly *uncalibrated* softmax,
+        # rather than a temperature nobody fitted.
+        self.assertEqual(cfg.temperature, 1.0)
         self.assertEqual(cfg.batch_size, 32)
         self.assertEqual(cfg.ece_target, 0.10)
         self.assertEqual(cfg.conformal_alpha, 0.10)
@@ -34,6 +37,7 @@ class TestConfigLoaderOverrides(unittest.TestCase):
         "CRE_LIBRARIAN_TOP_K_RETRIEVAL": "50",
         "CRE_LIBRARIAN_TOP_K_RERANK": "10",
         "CRE_LIBRARIAN_LINK_THRESHOLD": "0.7",
+        "CRE_LIBRARIAN_TEMPERATURE": "1.208",
         "CRE_LIBRARIAN_BATCH_SIZE": "64",
         "CRE_LIBRARIAN_ECE_TARGET": "0.05",
         "CRE_LIBRARIAN_CONFORMAL_ALPHA": "0.20",
@@ -47,6 +51,7 @@ def test_env_overrides_apply(self):
         self.assertEqual(cfg.top_k_retrieval, 50)
         self.assertEqual(cfg.top_k_rerank, 10)
         self.assertAlmostEqual(cfg.link_threshold, 0.7)
+        self.assertAlmostEqual(cfg.temperature, 1.208)
         self.assertEqual(cfg.batch_size, 64)
         self.assertAlmostEqual(cfg.ece_target, 0.05)
         self.assertAlmostEqual(cfg.conformal_alpha, 0.20)
@@ -65,6 +70,17 @@ def test_link_threshold_above_one_raises(self):
             with self.assertRaises(ValueError):
                 load_config()
 
+    def test_non_positive_temperature_raises(self):
+        """T divides the logits, so zero or negative is undefined, not merely a
+        bad setting — the same guard TemperatureScaler applies."""
+        for value in ("0", "-1.5", "nan"):
+            with self.subTest(value=value):
+                with mock.patch.dict(
+                    os.environ, {"CRE_LIBRARIAN_TEMPERATURE": value}, clear=True
+                ):
+                    with self.assertRaises(ValueError):
+                        load_config()
+
     def test_negative_top_k_retrieval_raises(self):
         with mock.patch.dict(
             os.environ, {"CRE_LIBRARIAN_TOP_K_RETRIEVAL": "-1"}, clear=True