feat(docs): Improve knowledgebase adherence

backend/app/crud/evaluations/fast.py

@@ -52,6 +52,8 @@
     calculate_cosine_similarity,
 )
 from app.crud.evaluations.judge import (
+    JudgeInputEnum,
+    JudgeMetricEnum,
     JudgeMetricSpec,
     JudgeResult,
     build_judge_params,
@@ -81,6 +83,7 @@
 from app.models.evaluation import RunModeEnum
 from app.models.llm.request import TextLLMParams
 from app.services.llm.mappers import map_kaapi_to_openai_params
+from app.services.response.response import get_file_search_results
 
 logger = logging.getLogger(__name__)
 
@@ -101,6 +104,21 @@
 UNSCOREABLE_EMPTY_GROUND_TRUTH = "empty_ground_truth"
 UNSCOREABLE_EMBEDDING_FAILED = "embedding_failed"
 
+# How many top KB matches to name in the knowledge_base trace comment.
+_KB_TOP_CHUNKS = 3
+
+
+def _format_top_kb_matches(sorted_chunks: list[dict[str, Any]]) -> str:
+    """Top-N retrieved chunks as 'biu-1.pdf (90.6%), faq.pdf (66.3%)'.
+
+    Expects chunks pre-sorted by score desc; old S3 payloads may lack filename.
+    """
+    matches = [
+        f"{c.get('filename') or 'unknown'} ({c.get('score', 0) * 100:.1f}%)"
+        for c in sorted_chunks
+    ]
+    return ", ".join(matches[:_KB_TOP_CHUNKS])
+
 
 # Per-call retry policy for Stage 1 / Stage 2.
 _RETRY_MAX_ATTEMPTS = 3
@@ -181,6 +199,7 @@ def _response_result(
     failed: bool,
     response_id: str | None = None,
     usage: dict[str, int] | None = None,
+    retrieved_chunks: list[dict[str, Any]] | None = None,
 ) -> dict[str, Any]:
     """One Stage-1 per-item result, in the batch path's shape."""
     return {
@@ -192,6 +211,7 @@ def _response_result(
         "usage": usage,
         "question_id": question_id,
         "failed": failed,
+        "retrieved_chunks": retrieved_chunks,
     }
 
 
@@ -254,6 +274,11 @@ def _responses_call_for_item(
             "total_tokens": int(_field(usage, "total_tokens", 0) or 0),
         },
         failed=False,
+        # Plain dicts (not FileResultChunk) so the unit stays JSON-serializable for S3.
+        retrieved_chunks=[
+            {"score": c.score, "text": c.text, "filename": c.filename}
+            for c in get_file_search_results(response)
+        ],
     )
 
 
@@ -491,6 +516,12 @@ def run_response_chunk(
             f"[run_response_chunk] {log_prefix} Mapper warnings: {mapper_warnings}"
         )
 
+    # Ask OpenAI to return the file_search hits so knowledge_base can judge them.
+    # tool_choice stays at the model default (auto) — consistent with normal calls;
+    # a row where the model doesn't query the KB is scored N/A, not forced to search.
+    if any(t.get("type") == "file_search" for t in base_params.get("tools", [])):
+        base_params["include"] = ["file_search_call.results"]
+
     results: list[dict[str, Any]] = []
     max_workers = max(
         1, min(settings.EVAL_FAST_API_CONCURRENCY, len(dataset_items_slice))
@@ -758,11 +789,10 @@ def _judge_rows(
     metrics = enabled_metric_specs()
 
     # Build base params once per run; judging is system-config only, so every metric
-    # uses its built-in prompt + fallback model.
+    # uses its built-in prompt + fallback model. The instructions vary per row (by
+    # applicable-metric subset) and are composed inside judge_row.
     try:
-        base_params, _system_prompt = build_judge_params(
-            session=session, metrics=metrics
-        )
+        base_params = build_judge_params(session=session)
     except Exception as exc:
         logger.error(
             f"[_judge_rows] {log_prefix} Judge setup failed; leaving all rows "
@@ -781,9 +811,18 @@ def _judge_rows(
                 openai_client=openai_client,
                 base_params=base_params,
                 metrics=metrics,
-                question=response.get("question", ""),
-                generated_answer=response.get("generated_output", ""),
-                golden_answer=response.get("ground_truth", ""),
+                inputs={
+                    JudgeInputEnum.QUESTION: response.get("question", ""),
+                    JudgeInputEnum.GENERATED_ANSWER: response.get(
+                        "generated_output", ""
+                    ),
+                    JudgeInputEnum.GOLDEN_ANSWER: response.get("ground_truth", ""),
+                    JudgeInputEnum.RETRIEVED_CHUNKS: "\n---\n".join(
+                        c.get("text", "")
+                        for c in (response.get("retrieved_chunks") or [])
+                        if c.get("text")
+                    ),
+                },
             ): (item_id, ref)
             for item_id, ref, response in judgeable
         }
@@ -824,8 +863,11 @@ def _attach_metric_scores(
         per_item[ref] = round(float(metric_score.score), 6)
         values.append(metric_score.score)
 
-    # Durable {ref: score} map on the metric's own column (Kaapi-native store).
-    setattr(eval_run, spec.per_item_column, per_item or None)
+    # Durable {ref: score} map on the metric's own column, when it has one. Metrics
+    # without a backup column (e.g. knowledge_base) rely on the score_trace_url trace
+    # unit, which carries the same per-row score plus its reasoning.
+    if spec.per_item_column:
+        setattr(eval_run, spec.per_item_column, per_item or None)
 
     if values:
         arr = np.array(values)
@@ -1094,18 +1136,48 @@ def _stage3_score_and_trace(
 
         judge_result = judge_results.get(item_id)
         if judge_result is not None:
+            sorted_chunks = sorted(
+                response.get("retrieved_chunks") or [],
+                key=lambda c: c.get("score", 0),
+                reverse=True,
+            )
+            top_matches = _format_top_kb_matches(sorted_chunks)
             for spec in enabled_metric_specs():
                 metric_score = judge_result.metrics.get(spec.key)
-                if metric_score is None:
-                    continue
-                trace_scores.append(
-                    {
-                        "name": spec.score_name,
-                        "value": round(metric_score.score, 2),
-                        "data_type": "NUMERIC",
-                        "comment": metric_score.reasoning,
-                    }
-                )
+                is_kb = spec.key == JudgeMetricEnum.KNOWLEDGE_BASE
+                if metric_score is not None:
+                    comment = metric_score.reasoning
+                    if is_kb:
+                        comment = f"{comment} | Top matches: {top_matches}"
+                    trace_scores.append(
+                        {
+                            "name": spec.score_name,
+                            "value": round(metric_score.score, 2),
+                            "data_type": "NUMERIC",
+                            "comment": comment,
+                        }
+                    )
+                elif is_kb:
+                    # KB dropped for this row: surface a human reason instead of a bare
+                    # N/A. Placeholder lives only in trace_scores, never the summary avg.
+                    if not sorted_chunks:
+                        # ponytail: empty chunks under auto tool_choice ~= not queried;
+                        # a "was queried" flag would disambiguate an empty-store hit, not
+                        # worth plumbing.
+                        reason = "Knowledge base not queried."
+                    else:
+                        # Chunks present but the judge returned no KB score (rare: a
+                        # well-formed reply that omitted the metric).
+                        reason = "Knowledge base score unavailable for this row."
+                    trace_scores.append(
+                        {
+                            "name": spec.score_name,
+                            "value": "N/A",
+                            "data_type": "CATEGORICAL",
+                            "comment": reason,
+                            "unscoreable": True,
+                        }
+                    )
 
         traces.append(
             {

backend/app/crud/evaluations/judge.py

@@ -29,6 +29,8 @@
     GROUND_TRUTH_SCORE_NAME,
     JUDGE_OUTPUT_INSTRUCTION,
     JUDGE_SYSTEM_PREAMBLE,
+    KNOWLEDGE_BASE_JUDGE_PROMPT,
+    KNOWLEDGE_BASE_SCORE_NAME,
 )
 from app.services.llm.mappers import map_kaapi_to_openai_params
 
@@ -39,20 +41,27 @@ class JudgeMetricEnum(str, Enum):
     """Registry keys — also the JSON keys the combined judge returns per metric."""
 
     GROUND_TRUTH = "ground_truth"
+    KNOWLEDGE_BASE = "knowledge_base"
 
 
 class JudgeInputEnum(str, Enum):
-    """Per-row inputs a metric may require in the composed judge prompt."""
+    """Per-row inputs a metric may require in the composed judge prompt.
+
+    Declaration order fixes the render order in the input block, so new inputs are
+    appended last to keep existing metrics' prompts byte-identical.
+    """
 
     QUESTION = "question"
     GENERATED_ANSWER = "generated_answer"
     GOLDEN_ANSWER = "golden_answer"
+    RETRIEVED_CHUNKS = "retrieved_chunks"
 
 
 _INPUT_LABELS: dict[JudgeInputEnum, str] = {
     JudgeInputEnum.QUESTION: "Question",
     JudgeInputEnum.GENERATED_ANSWER: "Generated answer",
     JudgeInputEnum.GOLDEN_ANSWER: "Golden (reference) answer",
+    JudgeInputEnum.RETRIEVED_CHUNKS: "Retrieved knowledge-base chunks",
 }
 
 
@@ -64,13 +73,15 @@ class JudgeMetricSpec:
     score_name: str
     prompt_fragment: str
     required_inputs: tuple[JudgeInputEnum, ...]
-    per_item_column: str
+    # None = no dedicated backup column; the per-row score + reasoning still land in
+    # the score_trace_url trace unit (the Kaapi-native source of truth).
+    per_item_column: str | None
     cost_stage: str
 
 
-# Phase 1: only ground_truth, knowledge_base / prompt slot in here. All
-# metrics are graded by one combined call, so they share a single judge model
-# (settings.EVAL_JUDGE_MODEL); there is no per-metric model.
+# prompt slots in here next. All metrics are graded by one combined call, so they
+# share a single judge model (settings.EVAL_JUDGE_MODEL); there is no per-metric
+# model. A metric only runs on a row that carries all its required_inputs.
 METRIC_REGISTRY: dict[JudgeMetricEnum, JudgeMetricSpec] = {
     JudgeMetricEnum.GROUND_TRUTH: JudgeMetricSpec(
         key=JudgeMetricEnum.GROUND_TRUTH,
@@ -84,6 +95,20 @@ class JudgeMetricSpec:
         per_item_column="per_item_ground_truth",
         cost_stage="ground_truth_judge",
     ),
+    JudgeMetricEnum.KNOWLEDGE_BASE: JudgeMetricSpec(
+        key=JudgeMetricEnum.KNOWLEDGE_BASE,
+        score_name=KNOWLEDGE_BASE_SCORE_NAME,
+        prompt_fragment=KNOWLEDGE_BASE_JUDGE_PROMPT,
+        # No QUESTION: groundedness judges the answer against the chunks alone.
+        required_inputs=(
+            JudgeInputEnum.GENERATED_ANSWER,
+            JudgeInputEnum.RETRIEVED_CHUNKS,
+        ),
+        # No backup column: the per-row groundedness score + reasoning live in the
+        # score_trace_url trace unit; v2 is fully Kaapi-native, so no resync store.
+        per_item_column=None,
+        cost_stage="knowledge_base_judge",
+    ),
 }
 
 
@@ -132,17 +157,13 @@ class JudgeResult:
     usage: dict[str, int]
 
 
-def build_judge_params(
-    *,
-    session: Session,
-    metrics: list[JudgeMetricSpec],
-) -> tuple[dict[str, Any], str]:
-    """Build the model-independent OpenAI body and the combined judge system prompt.
+def build_judge_params(*, session: Session) -> dict[str, Any]:
+    """Build the model-independent OpenAI body for the combined judge call.
 
-    Judging is system-config only: every metric uses its built-in rubric fragment,
-    and the single combined call runs on one shared judge model
-    (settings.EVAL_JUDGE_MODEL) for all metrics. The system prompt is the shared
-    preamble followed by each enabled metric's fragment.
+    Judging is system-config only: the single combined call runs on one shared
+    judge model (settings.EVAL_JUDGE_MODEL) for all metrics. `instructions` are NOT
+    baked here — they're the applicable-metric subset, which varies per row, so
+    `judge_row` composes and sets them from `_compose_system_prompt`.
     """
     judge_params: dict[str, Any] = {
         "model": settings.EVAL_JUDGE_MODEL,
@@ -155,11 +176,28 @@ def build_judge_params(
     if mapper_warnings:
         logger.warning(f"[build_judge_params] Mapper warnings: {mapper_warnings}")
 
-    fragments = "\n\n".join(spec.prompt_fragment for spec in metrics)
-    system_prompt = f"{JUDGE_SYSTEM_PREAMBLE}\n\n{fragments}"
-    # The judge prompt IS the instructions; overwrite anything the mapper carried.
-    base_params["instructions"] = system_prompt
-    return base_params, system_prompt
+    return base_params
+
+
+def _compose_system_prompt(metrics: list[JudgeMetricSpec]) -> str:
+    """Shared preamble followed by each metric's built-in rubric fragment."""
+    fragments = [spec.prompt_fragment for spec in metrics]
+    return f"{JUDGE_SYSTEM_PREAMBLE}\n\n" + "\n\n".join(fragments)
+
+
+def _applicable_metrics(
+    metrics: list[JudgeMetricSpec], inputs: dict[JudgeInputEnum, str]
+) -> list[JudgeMetricSpec]:
+    """The metrics whose required_inputs are all present and non-empty for this row.
+
+    A row missing an input (e.g. no retrieved chunks) simply omits that metric from
+    the judge call — it stays unscoreable for that row, not scored 0.
+    """
+    return [
+        spec
+        for spec in metrics
+        if all(inputs.get(key, "").strip() for key in spec.required_inputs)
+    ]
 
 
 def _compose_judge_input(
@@ -273,27 +311,27 @@ def judge_row(
     openai_client: OpenAI,
     base_params: dict[str, Any],
     metrics: list[JudgeMetricSpec],
-    question: str,
-    generated_answer: str,
-    golden_answer: str,
+    inputs: dict[JudgeInputEnum, str],
 ) -> JudgeResult:
     """Run one combined judge completion for a row and return its per-metric result.
 
     `base_params` is the model-independent body from `build_judge_params`, built
-    once per run; only `input` varies per row. Raises on retry-exhausted OpenAI
-    errors or malformed output so the caller can flag the whole row unscoreable.
+    once per run; the system prompt and input block are composed here from the
+    per-row applicable-metric subset (a metric whose inputs are absent is dropped,
+    so its key never enters the prompt or the expected reply). Raises on
+    retry-exhausted OpenAI errors or malformed output so the caller can flag the
+    whole row unscoreable.
     """
+    applicable = _applicable_metrics(metrics, inputs)
+    if not applicable:
+        raise ValueError("no judge metric applies to this row's inputs")
+
     model = base_params.get("model")
     params = {
         **base_params,
-        "input": _compose_judge_input(
-            metrics=metrics,
-            inputs={
-                JudgeInputEnum.QUESTION: question,
-                JudgeInputEnum.GENERATED_ANSWER: generated_answer,
-                JudgeInputEnum.GOLDEN_ANSWER: golden_answer,
-            },
-        ),
+        # The judge prompt IS the instructions; overwrite anything the mapper carried.
+        "instructions": _compose_system_prompt(applicable),
+        "input": _compose_judge_input(metrics=applicable, inputs=inputs),
     }
 
     try:
@@ -319,5 +357,5 @@ def judge_row(
         "total_tokens": int(getattr(usage_obj, "total_tokens", 0) or 0),
     }
 
-    metric_scores = _parse_judge_output(_extract_response_text(response), metrics)
+    metric_scores = _parse_judge_output(_extract_response_text(response), applicable)
     return JudgeResult(metrics=metric_scores, usage=usage)