From b02bcc83e3376c40bd8559a7fbf938e2bc8d0556 Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Mon, 20 Jul 2026 20:47:51 +0530 Subject: [PATCH 1/2] first stab at running evals --- backend/app/crud/evaluations/fast.py | 42 +++- backend/app/crud/evaluations/judge.py | 106 +++++--- backend/app/crud/evaluations/score.py | 20 ++ .../tests/crud/evaluations/test_fast_judge.py | 123 +++++++++- .../app/tests/crud/evaluations/test_judge.py | 226 +++++++++++++++--- docs/wiki/modules/evaluations.md | 6 +- 6 files changed, 449 insertions(+), 74 deletions(-) diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index 24d8f364d..33079a85f 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -52,6 +52,7 @@ calculate_cosine_similarity, ) from app.crud.evaluations.judge import ( + JudgeInputEnum, JudgeMetricSpec, JudgeResult, build_judge_params, @@ -81,6 +82,7 @@ from app.models.evaluation import RunModeEnum from app.models.llm.request import TextLLMParams from app.services.llm.mappers import map_kaapi_to_openai_params +from app.services.response.response import get_file_search_results logger = logging.getLogger(__name__) @@ -181,6 +183,7 @@ def _response_result( failed: bool, response_id: str | None = None, usage: dict[str, int] | None = None, + retrieved_chunks: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: """One Stage-1 per-item result, in the batch path's shape.""" return { @@ -192,6 +195,7 @@ def _response_result( "usage": usage, "question_id": question_id, "failed": failed, + "retrieved_chunks": retrieved_chunks, } @@ -254,6 +258,11 @@ def _responses_call_for_item( "total_tokens": int(_field(usage, "total_tokens", 0) or 0), }, failed=False, + # Plain dicts (not FileResultChunk) so the unit stays JSON-serializable for S3. + retrieved_chunks=[ + {"score": c.score, "text": c.text} + for c in get_file_search_results(response) + ], ) @@ -491,6 +500,10 @@ def run_response_chunk( f"[run_response_chunk] {log_prefix} Mapper warnings: {mapper_warnings}" ) + # Ask OpenAI to return the file_search hits so knowledge_base can judge them. + if any(t.get("type") == "file_search" for t in base_params.get("tools", [])): + base_params["include"] = ["file_search_call.results"] + results: list[dict[str, Any]] = [] max_workers = max( 1, min(settings.EVAL_FAST_API_CONCURRENCY, len(dataset_items_slice)) @@ -758,11 +771,10 @@ def _judge_rows( metrics = enabled_metric_specs() # Build base params once per run; judging is system-config only, so every metric - # uses its built-in prompt + fallback model. + # uses its built-in prompt + fallback model. The instructions vary per row (by + # applicable-metric subset) and are composed inside judge_row. try: - base_params, _system_prompt = build_judge_params( - session=session, metrics=metrics - ) + base_params = build_judge_params(session=session) except Exception as exc: logger.error( f"[_judge_rows] {log_prefix} Judge setup failed; leaving all rows " @@ -781,9 +793,18 @@ def _judge_rows( openai_client=openai_client, base_params=base_params, metrics=metrics, - question=response.get("question", ""), - generated_answer=response.get("generated_output", ""), - golden_answer=response.get("ground_truth", ""), + inputs={ + JudgeInputEnum.QUESTION: response.get("question", ""), + JudgeInputEnum.GENERATED_ANSWER: response.get( + "generated_output", "" + ), + JudgeInputEnum.GOLDEN_ANSWER: response.get("ground_truth", ""), + JudgeInputEnum.RETRIEVED_CHUNKS: "\n---\n".join( + c.get("text", "") + for c in (response.get("retrieved_chunks") or []) + if c.get("text") + ), + }, ): (item_id, ref) for item_id, ref, response in judgeable } @@ -824,8 +845,11 @@ def _attach_metric_scores( per_item[ref] = round(float(metric_score.score), 6) values.append(metric_score.score) - # Durable {ref: score} map on the metric's own column (Kaapi-native store). - setattr(eval_run, spec.per_item_column, per_item or None) + # Durable {ref: score} map on the metric's own column, when it has one. Metrics + # without a backup column (e.g. knowledge_base) rely on the score_trace_url trace + # unit, which carries the same per-row score plus its reasoning. + if spec.per_item_column: + setattr(eval_run, spec.per_item_column, per_item or None) if values: arr = np.array(values) diff --git a/backend/app/crud/evaluations/judge.py b/backend/app/crud/evaluations/judge.py index f2fe979d9..1f34eaa60 100644 --- a/backend/app/crud/evaluations/judge.py +++ b/backend/app/crud/evaluations/judge.py @@ -29,6 +29,8 @@ GROUND_TRUTH_SCORE_NAME, JUDGE_OUTPUT_INSTRUCTION, JUDGE_SYSTEM_PREAMBLE, + KNOWLEDGE_BASE_JUDGE_PROMPT, + KNOWLEDGE_BASE_SCORE_NAME, ) from app.services.llm.mappers import map_kaapi_to_openai_params @@ -39,20 +41,27 @@ class JudgeMetricEnum(str, Enum): """Registry keys — also the JSON keys the combined judge returns per metric.""" GROUND_TRUTH = "ground_truth" + KNOWLEDGE_BASE = "knowledge_base" class JudgeInputEnum(str, Enum): - """Per-row inputs a metric may require in the composed judge prompt.""" + """Per-row inputs a metric may require in the composed judge prompt. + + Declaration order fixes the render order in the input block, so new inputs are + appended last to keep existing metrics' prompts byte-identical. + """ QUESTION = "question" GENERATED_ANSWER = "generated_answer" GOLDEN_ANSWER = "golden_answer" + RETRIEVED_CHUNKS = "retrieved_chunks" _INPUT_LABELS: dict[JudgeInputEnum, str] = { JudgeInputEnum.QUESTION: "Question", JudgeInputEnum.GENERATED_ANSWER: "Generated answer", JudgeInputEnum.GOLDEN_ANSWER: "Golden (reference) answer", + JudgeInputEnum.RETRIEVED_CHUNKS: "Retrieved knowledge-base chunks", } @@ -64,13 +73,15 @@ class JudgeMetricSpec: score_name: str prompt_fragment: str required_inputs: tuple[JudgeInputEnum, ...] - per_item_column: str + # None = no dedicated backup column; the per-row score + reasoning still land in + # the score_trace_url trace unit (the Kaapi-native source of truth). + per_item_column: str | None cost_stage: str -# Phase 1: only ground_truth, knowledge_base / prompt slot in here. All -# metrics are graded by one combined call, so they share a single judge model -# (settings.EVAL_JUDGE_MODEL); there is no per-metric model. +# prompt slots in here next. All metrics are graded by one combined call, so they +# share a single judge model (settings.EVAL_JUDGE_MODEL); there is no per-metric +# model. A metric only runs on a row that carries all its required_inputs. METRIC_REGISTRY: dict[JudgeMetricEnum, JudgeMetricSpec] = { JudgeMetricEnum.GROUND_TRUTH: JudgeMetricSpec( key=JudgeMetricEnum.GROUND_TRUTH, @@ -84,6 +95,20 @@ class JudgeMetricSpec: per_item_column="per_item_ground_truth", cost_stage="ground_truth_judge", ), + JudgeMetricEnum.KNOWLEDGE_BASE: JudgeMetricSpec( + key=JudgeMetricEnum.KNOWLEDGE_BASE, + score_name=KNOWLEDGE_BASE_SCORE_NAME, + prompt_fragment=KNOWLEDGE_BASE_JUDGE_PROMPT, + # No QUESTION: groundedness judges the answer against the chunks alone. + required_inputs=( + JudgeInputEnum.GENERATED_ANSWER, + JudgeInputEnum.RETRIEVED_CHUNKS, + ), + # No backup column: the per-row groundedness score + reasoning live in the + # score_trace_url trace unit; v2 is fully Kaapi-native, so no resync store. + per_item_column=None, + cost_stage="knowledge_base_judge", + ), } @@ -132,17 +157,13 @@ class JudgeResult: usage: dict[str, int] -def build_judge_params( - *, - session: Session, - metrics: list[JudgeMetricSpec], -) -> tuple[dict[str, Any], str]: - """Build the model-independent OpenAI body and the combined judge system prompt. +def build_judge_params(*, session: Session) -> dict[str, Any]: + """Build the model-independent OpenAI body for the combined judge call. - Judging is system-config only: every metric uses its built-in rubric fragment, - and the single combined call runs on one shared judge model - (settings.EVAL_JUDGE_MODEL) for all metrics. The system prompt is the shared - preamble followed by each enabled metric's fragment. + Judging is system-config only: the single combined call runs on one shared + judge model (settings.EVAL_JUDGE_MODEL) for all metrics. `instructions` are NOT + baked here — they're the applicable-metric subset, which varies per row, so + `judge_row` composes and sets them from `_compose_system_prompt`. """ judge_params: dict[str, Any] = { "model": settings.EVAL_JUDGE_MODEL, @@ -155,11 +176,28 @@ def build_judge_params( if mapper_warnings: logger.warning(f"[build_judge_params] Mapper warnings: {mapper_warnings}") - fragments = "\n\n".join(spec.prompt_fragment for spec in metrics) - system_prompt = f"{JUDGE_SYSTEM_PREAMBLE}\n\n{fragments}" - # The judge prompt IS the instructions; overwrite anything the mapper carried. - base_params["instructions"] = system_prompt - return base_params, system_prompt + return base_params + + +def _compose_system_prompt(metrics: list[JudgeMetricSpec]) -> str: + """Shared preamble followed by each metric's built-in rubric fragment.""" + fragments = [spec.prompt_fragment for spec in metrics] + return f"{JUDGE_SYSTEM_PREAMBLE}\n\n" + "\n\n".join(fragments) + + +def _applicable_metrics( + metrics: list[JudgeMetricSpec], inputs: dict[JudgeInputEnum, str] +) -> list[JudgeMetricSpec]: + """The metrics whose required_inputs are all present and non-empty for this row. + + A row missing an input (e.g. no retrieved chunks) simply omits that metric from + the judge call — it stays unscoreable for that row, not scored 0. + """ + return [ + spec + for spec in metrics + if all(inputs.get(key, "").strip() for key in spec.required_inputs) + ] def _compose_judge_input( @@ -273,27 +311,27 @@ def judge_row( openai_client: OpenAI, base_params: dict[str, Any], metrics: list[JudgeMetricSpec], - question: str, - generated_answer: str, - golden_answer: str, + inputs: dict[JudgeInputEnum, str], ) -> JudgeResult: """Run one combined judge completion for a row and return its per-metric result. `base_params` is the model-independent body from `build_judge_params`, built - once per run; only `input` varies per row. Raises on retry-exhausted OpenAI - errors or malformed output so the caller can flag the whole row unscoreable. + once per run; the system prompt and input block are composed here from the + per-row applicable-metric subset (a metric whose inputs are absent is dropped, + so its key never enters the prompt or the expected reply). Raises on + retry-exhausted OpenAI errors or malformed output so the caller can flag the + whole row unscoreable. """ + applicable = _applicable_metrics(metrics, inputs) + if not applicable: + raise ValueError("no judge metric applies to this row's inputs") + model = base_params.get("model") params = { **base_params, - "input": _compose_judge_input( - metrics=metrics, - inputs={ - JudgeInputEnum.QUESTION: question, - JudgeInputEnum.GENERATED_ANSWER: generated_answer, - JudgeInputEnum.GOLDEN_ANSWER: golden_answer, - }, - ), + # The judge prompt IS the instructions; overwrite anything the mapper carried. + "instructions": _compose_system_prompt(applicable), + "input": _compose_judge_input(metrics=applicable, inputs=inputs), } try: @@ -319,5 +357,5 @@ def judge_row( "total_tokens": int(getattr(usage_obj, "total_tokens", 0) or 0), } - metric_scores = _parse_judge_output(_extract_response_text(response), metrics) + metric_scores = _parse_judge_output(_extract_response_text(response), applicable) return JudgeResult(metrics=metric_scores, usage=usage) diff --git a/backend/app/crud/evaluations/score.py b/backend/app/crud/evaluations/score.py index bd00782d8..fd3de1d4a 100644 --- a/backend/app/crud/evaluations/score.py +++ b/backend/app/crud/evaluations/score.py @@ -17,6 +17,7 @@ ) GROUND_TRUTH_SCORE_NAME: str = "Adherence to Ground Truth" +KNOWLEDGE_BASE_SCORE_NAME: str = "Adherence to Knowledge Base" JUDGE_FAILED_REASON: str = "judge_failed" # Reasons an item cannot be scored, recorded in EvaluationRun.unscoreable. @@ -52,6 +53,25 @@ "Reasoning: name what was correct or what was missing/contradicted." ) +KNOWLEDGE_BASE_JUDGE_PROMPT: str = ( + 'Adherence to Knowledge Base (score key "knowledge_base"):\n' + "Judge ONLY whether the answer's claims are supported by the retrieved " + "knowledge-base chunks (groundedness / hallucination detection).\n" + "- Break the answer into its distinct factual claims. A claim is supported iff " + "it is stated in, or directly entailed by, at least one chunk.\n" + "- Score = supported claims / total claims: all supported = 1.0; mostly " + "unsupported trends toward 0.0.\n" + "- Generic pleasantries or a safe refusal that make no factual claim are " + "trivially grounded — score them high.\n" + "- Judge groundedness ONLY, not correctness, completeness, or " + "instruction-following. A claim faithful to the chunks is grounded even if the " + "chunks are themselves wrong.\n" + "- Do NOT use any outside knowledge; the retrieved chunks are the ONLY allowed " + "source of support.\n" + "Reasoning: when the score is below 1.0, name the specific unsupported or " + "invented claim." +) + JUDGE_OUTPUT_INSTRUCTION: str = ( "Respond with ONLY a single JSON object mapping each metric key to its result, of " 'the form {{"": {{"score": , "reasoning": ' diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index 0e3a2ff82..c49066b54 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -21,6 +21,7 @@ from typing import Any from unittest.mock import MagicMock, patch +import openai import pytest from sqlmodel import Session @@ -29,13 +30,19 @@ CHUNK_CONFIG_INDEX, CHUNK_CONFIG_RUN_ID, JOB_TYPE_EVALUATION_FAST_CHUNK, + _responses_call_for_item, run_fast_evaluation, ) -from app.crud.evaluations.score import GROUND_TRUTH_SCORE_NAME, JUDGE_FAILED_REASON +from app.crud.evaluations.score import ( + GROUND_TRUTH_SCORE_NAME, + JUDGE_FAILED_REASON, + KNOWLEDGE_BASE_SCORE_NAME, +) from app.models import Config, EvaluationDataset, EvaluationRun from app.models.batch_job import BatchJob from app.models.evaluation import RunModeEnum from app.models.llm.request import ConfigBlob, KaapiCompletionConfig +from app.models.response import FileResultChunk from app.tests.utils.auth import TestAuthContext from app.tests.utils.test_data import ( create_test_config, @@ -459,3 +466,117 @@ def _judge(params): summary_names = {s["name"] for s in result.score["summary_scores"]} assert GROUND_TRUTH_SCORE_NAME not in summary_names assert COSINE_SCORE_NAME in summary_names + + +def _responses_item(item_id: str = "item-1") -> dict[str, Any]: + return { + "id": item_id, + "input": {"question": "What is X?"}, + "expected_output": {"answer": "golden"}, + "metadata": {"question_id": 1}, + } + + +def _openai_response(): + return SimpleNamespace( + output_text="generated answer", + output=[], + id="resp_1", + usage=SimpleNamespace(input_tokens=5, output_tokens=5, total_tokens=10), + ) + + +class TestResponsesChunkCapture: + """`_responses_call_for_item` flattens file_search hits into JSON-safe dicts.""" + + def test_success_with_chunks_returns_serializable_plain_dicts(self): + client = MagicMock() + client.responses.create.return_value = _openai_response() + with patch( + "app.crud.evaluations.fast.get_file_search_results", + return_value=[ + FileResultChunk(score=0.91, text="chunk A"), + FileResultChunk(score=0.42, text="chunk B"), + ], + ): + result = _responses_call_for_item( + openai_client=client, + base_params={"model": "gpt-4o"}, + item=_responses_item(), + ) + + assert result["failed"] is False + assert result["retrieved_chunks"] == [ + {"score": 0.91, "text": "chunk A"}, + {"score": 0.42, "text": "chunk B"}, + ] + json.dumps(result) # the S3 unit must stay JSON-serializable + + def test_success_without_hits_returns_empty_chunks(self): + client = MagicMock() + client.responses.create.return_value = _openai_response() + with patch( + "app.crud.evaluations.fast.get_file_search_results", return_value=[] + ): + result = _responses_call_for_item( + openai_client=client, + base_params={"model": "gpt-4o"}, + item=_responses_item(), + ) + + assert result["failed"] is False + assert result["retrieved_chunks"] == [] + + def test_error_path_has_no_chunks(self): + client = MagicMock() + client.responses.create.side_effect = openai.OpenAIError("provider down") + with patch("app.crud.evaluations.fast.get_file_search_results") as fake_search: + result = _responses_call_for_item( + openai_client=client, + base_params={"model": "gpt-4o"}, + item=_responses_item(), + ) + + assert result["failed"] is True + assert result["retrieved_chunks"] is None + fake_search.assert_not_called() + + +class TestKnowledgeBaseScoring: + def test_groundedness_scores_only_chunked_rows( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + """A chunk-less row is unscoreable for groundedness (absent, not 0), while + ground truth still scores every judged row.""" + eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True) + chunked = _resp_result("item-chunked", "Q1", "golden-1") + chunked["retrieved_chunks"] = [{"score": 0.9, "text": "supporting chunk"}] + plain = _resp_result("item-plain", "Q2", "golden-2") # no retrieved_chunks + _seed_chunk(db=db, eval_run=eval_run, results=[chunked, plain], store=_s3_store) + + def _judge(params): + if KNOWLEDGE_BASE_SCORE_NAME in params["instructions"]: + return _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.8, "reasoning": "gt"}, + "knowledge_base": {"score": 0.6, "reasoning": "kb"}, + } + ) + ) + return _judge_response(0.9, "gt only") + + result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) + + assert result.status == "completed" + run = db.get(EvaluationRun, result.id) + # ground_truth keeps its durable per-row column; knowledge_base has none — + # its per-row score lives only in the score_trace_url trace unit below. + assert set(run.per_item_ground_truth) == {"item-chunked", "item-plain"} + + traces = _trace_by_ref(result) + kb_chunked = _score_named(traces["item-chunked"], KNOWLEDGE_BASE_SCORE_NAME) + assert kb_chunked is not None + assert kb_chunked["value"] == 0.6 + assert _score_named(traces["item-plain"], KNOWLEDGE_BASE_SCORE_NAME) is None + assert _score_named(traces["item-plain"], GROUND_TRUTH_SCORE_NAME) is not None diff --git a/backend/app/tests/crud/evaluations/test_judge.py b/backend/app/tests/crud/evaluations/test_judge.py index 95bff9046..fd6ce064c 100644 --- a/backend/app/tests/crud/evaluations/test_judge.py +++ b/backend/app/tests/crud/evaluations/test_judge.py @@ -18,10 +18,13 @@ from app.core.config import settings from app.crud.evaluations.judge import ( + METRIC_REGISTRY, JudgeInputEnum, JudgeMetricEnum, MetricScore, + _applicable_metrics, _compose_judge_input, + _compose_system_prompt, _parse_judge_output, build_judge_params, enabled_metric_specs, @@ -31,8 +34,13 @@ GROUND_TRUTH_JUDGE_PROMPT, GROUND_TRUTH_SCORE_NAME, JUDGE_SYSTEM_PREAMBLE, + KNOWLEDGE_BASE_JUDGE_PROMPT, + KNOWLEDGE_BASE_SCORE_NAME, ) +_GROUND_TRUTH_SPEC = METRIC_REGISTRY[JudgeMetricEnum.GROUND_TRUTH] +_KNOWLEDGE_BASE_SPEC = METRIC_REGISTRY[JudgeMetricEnum.KNOWLEDGE_BASE] + def _judge_response(payload: dict | str, *, usage=(15, 8, 23)): """Mimic the OpenAI Responses SDK object the judge parses (`output_text` + usage).""" @@ -87,6 +95,34 @@ def test_drops_only_the_missing_metric_when_others_present(self) -> None: ) assert list(result) == [JudgeMetricEnum.GROUND_TRUTH] + def test_parses_both_metrics_when_both_requested(self) -> None: + result = _parse_judge_output( + json.dumps( + { + "ground_truth": {"score": 0.8, "reasoning": "same facts"}, + "knowledge_base": { + "score": 0.6, + "reasoning": "one unsupported claim", + }, + } + ), + [_GROUND_TRUTH_SPEC, _KNOWLEDGE_BASE_SPEC], + ) + assert set(result) == { + JudgeMetricEnum.GROUND_TRUTH, + JudgeMetricEnum.KNOWLEDGE_BASE, + } + assert result[JudgeMetricEnum.KNOWLEDGE_BASE].score == 0.6 + + def test_missing_knowledge_base_leaves_only_ground_truth(self) -> None: + # Both metrics requested but the reply omits knowledge_base: that metric is + # silently unscoreable for the row — no raise, ground_truth still parsed. + result = _parse_judge_output( + json.dumps({"ground_truth": {"score": 0.9, "reasoning": "correct"}}), + [_GROUND_TRUTH_SPEC, _KNOWLEDGE_BASE_SPEC], + ) + assert set(result) == {JudgeMetricEnum.GROUND_TRUTH} + def test_raises_when_no_enabled_metric_scored(self) -> None: specs = enabled_metric_specs() with pytest.raises(ValueError, match="scored no enabled metric"): @@ -141,6 +177,52 @@ def test_input_contains_all_three_ground_truth_inputs(self) -> None: assert "ground_truth" in composed +class TestApplicableMetrics: + """A metric runs on a row only when all its required inputs are present + non-empty.""" + + def test_no_chunks_yields_ground_truth_only(self) -> None: + applicable = _applicable_metrics( + enabled_metric_specs(), + { + JudgeInputEnum.QUESTION: "Q", + JudgeInputEnum.GENERATED_ANSWER: "A", + JudgeInputEnum.GOLDEN_ANSWER: "G", + JudgeInputEnum.RETRIEVED_CHUNKS: "", + }, + ) + assert [s.key for s in applicable] == [JudgeMetricEnum.GROUND_TRUTH] + + def test_chunks_present_yields_both_metrics(self) -> None: + applicable = _applicable_metrics( + enabled_metric_specs(), + { + JudgeInputEnum.QUESTION: "Q", + JudgeInputEnum.GENERATED_ANSWER: "A", + JudgeInputEnum.GOLDEN_ANSWER: "G", + JudgeInputEnum.RETRIEVED_CHUNKS: "chunk text", + }, + ) + assert [s.key for s in applicable] == [ + JudgeMetricEnum.GROUND_TRUTH, + JudgeMetricEnum.KNOWLEDGE_BASE, + ] + + def test_empty_question_drops_ground_truth(self) -> None: + applicable = _applicable_metrics( + enabled_metric_specs(), + { + JudgeInputEnum.QUESTION: "", + JudgeInputEnum.GENERATED_ANSWER: "A", + JudgeInputEnum.GOLDEN_ANSWER: "G", + JudgeInputEnum.RETRIEVED_CHUNKS: "chunk text", + }, + ) + # QUESTION is empty, so ground_truth is inapplicable; knowledge_base (no QUESTION + # requirement) still applies from the generated answer + chunks. + assert JudgeMetricEnum.GROUND_TRUTH not in [s.key for s in applicable] + assert [s.key for s in applicable] == [JudgeMetricEnum.KNOWLEDGE_BASE] + + class TestBuildJudgeParams: """FR-9: system-config judging uses the fallback model + built-in ground-truth prompt. @@ -149,28 +231,37 @@ class TestBuildJudgeParams: """ def test_defaults_to_fallback_model_and_builtin_prompt(self, db: Session) -> None: - specs = enabled_metric_specs() - base_params, system_prompt = build_judge_params(session=db, metrics=specs) + base_params = build_judge_params(session=db) assert base_params["model"] == settings.EVAL_JUDGE_MODEL assert base_params["model"] == "gpt-5-mini" assert "temperature" not in base_params + # Instructions are the per-row applicable-metric subset, so they are composed + # in judge_row, never baked into the shared base params. + assert "instructions" not in base_params + + system_prompt = _compose_system_prompt(enabled_metric_specs()) assert JUDGE_SYSTEM_PREAMBLE in system_prompt assert GROUND_TRUTH_JUDGE_PROMPT in system_prompt - # The judge prompt IS the instructions; a bot's own instructions never leak. - assert base_params["instructions"] == system_prompt class TestJudgeRow: """FR-3 / FR-15: one combined call per row, returning scores + usage; raises isolate.""" def _base_params(self, db: Session) -> dict: - specs = enabled_metric_specs() - base_params, _ = build_judge_params(session=db, metrics=specs) - return base_params + return build_judge_params(session=db) + + @staticmethod + def _gt_inputs( + question: str = "Q", generated: str = "A", golden: str = "G" + ) -> dict[JudgeInputEnum, str]: + return { + JudgeInputEnum.QUESTION: question, + JudgeInputEnum.GENERATED_ANSWER: generated, + JudgeInputEnum.GOLDEN_ANSWER: golden, + } def test_returns_metric_score_and_usage(self, db: Session) -> None: - specs = enabled_metric_specs() with patch( "app.crud.evaluations.judge._create_judge_response", return_value=_judge_response( @@ -180,10 +271,8 @@ def test_returns_metric_score_and_usage(self, db: Session) -> None: result = judge_row( openai_client=SimpleNamespace(), base_params=self._base_params(db), - metrics=specs, - question="Q", - generated_answer="A", - golden_answer="A-golden", + metrics=enabled_metric_specs(), + inputs=self._gt_inputs(generated="A", golden="A-golden"), ) assert result.metrics[JudgeMetricEnum.GROUND_TRUTH] == MetricScore( @@ -195,6 +284,82 @@ def test_returns_metric_score_and_usage(self, db: Session) -> None: "total_tokens": 23, } + def test_chunkless_row_judges_ground_truth_only(self, db: Session) -> None: + captured: dict = {} + + def _capture(_client, params): + captured.update(params) + return _judge_response({"ground_truth": {"score": 0.9, "reasoning": "ok"}}) + + with patch( + "app.crud.evaluations.judge._create_judge_response", side_effect=_capture + ): + result = judge_row( + openai_client=SimpleNamespace(), + base_params=self._base_params(db), + metrics=enabled_metric_specs(), + inputs=self._gt_inputs(), + ) + + assert set(result.metrics) == {JudgeMetricEnum.GROUND_TRUTH} + # With no chunks, knowledge_base is dropped: its rubric, chunk label, and key + # must never reach the judge call for this row. + assert KNOWLEDGE_BASE_JUDGE_PROMPT not in captured["instructions"] + assert "Retrieved knowledge-base chunks" not in captured["input"] + assert "knowledge_base" not in captured["input"] + + def test_chunk_present_row_judges_both_metrics(self, db: Session) -> None: + with patch( + "app.crud.evaluations.judge._create_judge_response", + return_value=_judge_response( + { + "ground_truth": {"score": 0.8, "reasoning": "gt"}, + "knowledge_base": {"score": 0.7, "reasoning": "kb"}, + } + ), + ): + result = judge_row( + openai_client=SimpleNamespace(), + base_params=self._base_params(db), + metrics=enabled_metric_specs(), + inputs={ + **self._gt_inputs(), + JudgeInputEnum.RETRIEVED_CHUNKS: "supporting chunk", + }, + ) + + assert set(result.metrics) == { + JudgeMetricEnum.GROUND_TRUTH, + JudgeMetricEnum.KNOWLEDGE_BASE, + } + assert result.metrics[JudgeMetricEnum.KNOWLEDGE_BASE].score == 0.7 + + def test_chunkless_prompt_is_byte_identical_to_ground_truth_only( + self, db: Session + ) -> None: + """Adding knowledge_base to the registry leaves the chunk-less path unchanged.""" + captured: dict = {} + + def _capture(_client, params): + captured.update(params) + return _judge_response({"ground_truth": {"score": 0.5, "reasoning": "x"}}) + + with patch( + "app.crud.evaluations.judge._create_judge_response", side_effect=_capture + ): + judge_row( + openai_client=SimpleNamespace(), + base_params=self._base_params(db), + metrics=enabled_metric_specs(), + inputs=self._gt_inputs(question="Qx", generated="Ax", golden="Gx"), + ) + + assert captured["instructions"] == _compose_system_prompt([_GROUND_TRUTH_SPEC]) + assert "Question:\nQx" in captured["input"] + assert "Generated answer:\nAx" in captured["input"] + assert "Golden (reference) answer:\nGx" in captured["input"] + assert "Include exactly these metric keys: ground_truth." in captured["input"] + def test_judge_call_receives_question_answer_and_golden(self, db: Session) -> None: """FR-3: the row's golden answer reaches the judge completion input.""" captured: dict = {} @@ -212,9 +377,11 @@ def _capture(_client, params): openai_client=SimpleNamespace(), base_params=self._base_params(db), metrics=enabled_metric_specs(), - question="Who wrote Hamlet?", - generated_answer="Shakespeare wrote it.", - golden_answer="William Shakespeare", + inputs=self._gt_inputs( + question="Who wrote Hamlet?", + generated="Shakespeare wrote it.", + golden="William Shakespeare", + ), ) assert "Who wrote Hamlet?" in captured["input"] @@ -231,9 +398,7 @@ def test_malformed_output_raises_for_row_isolation(self, db: Session) -> None: openai_client=SimpleNamespace(), base_params=self._base_params(db), metrics=enabled_metric_specs(), - question="Q", - generated_answer="A", - golden_answer="G", + inputs=self._gt_inputs(), ) def test_openai_error_propagates_for_row_isolation(self, db: Session) -> None: @@ -246,16 +411,23 @@ def test_openai_error_propagates_for_row_isolation(self, db: Session) -> None: openai_client=SimpleNamespace(), base_params=self._base_params(db), metrics=enabled_metric_specs(), - question="Q", - generated_answer="A", - golden_answer="G", + inputs=self._gt_inputs(), ) -def test_ground_truth_is_the_only_enabled_metric() -> None: - """Phase 1 ships exactly one metric; guards against silently enabling more.""" +def test_enabled_metrics_are_ground_truth_and_knowledge_base() -> None: + """v2 ships exactly the two judge metrics; guards against silently enabling more.""" specs = enabled_metric_specs() - assert [s.key for s in specs] == [JudgeMetricEnum.GROUND_TRUTH] - assert specs[0].score_name == GROUND_TRUTH_SCORE_NAME - assert specs[0].per_item_column == "per_item_ground_truth" - assert specs[0].cost_stage == "ground_truth_judge" + assert [s.key for s in specs] == [ + JudgeMetricEnum.GROUND_TRUTH, + JudgeMetricEnum.KNOWLEDGE_BASE, + ] + + assert _GROUND_TRUTH_SPEC.score_name == GROUND_TRUTH_SCORE_NAME + assert _GROUND_TRUTH_SPEC.per_item_column == "per_item_ground_truth" + assert _GROUND_TRUTH_SPEC.cost_stage == "ground_truth_judge" + + assert _KNOWLEDGE_BASE_SPEC.score_name == KNOWLEDGE_BASE_SCORE_NAME + # No backup column: the per-row groundedness score lives in score_trace_url. + assert _KNOWLEDGE_BASE_SPEC.per_item_column is None + assert _KNOWLEDGE_BASE_SPEC.cost_stage == "knowledge_base_judge" diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index eaea4127e..474caa9bd 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -20,12 +20,12 @@ All paths relative to `backend/app/`. | `batch_job` (BatchJob) | `models/batch_job.py` | Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. -v2 judge fields on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip), `per_item_ground_truth` (JSONB `{ref: score}`, Kaapi-native, not synced to Langfuse). Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompt, no per-run config. +v2 judge fields on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip), `per_item_ground_truth` (JSONB `{ref: score}`, Kaapi-native, not synced to Langfuse) backing the `ground_truth` metric. The `knowledge_base` metric has no backup column — its per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth). Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompt, no per-run config. ## Services / CRUD - `services/evaluations/` — `evaluation.py`, `dataset.py`, `fast.py`, `judge.py` (v2 judged run trigger), `batch_job.py`, `validators.py`, `prompt_improvement.py` - `services/stt_evaluations/`, `services/tts_evaluations/` -- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (metric registry + combined judge call), `score.py`, `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py` +- `crud/evaluations/` — `core.py`, `batch.py`, `fast.py`, `judge.py` (`METRIC_REGISTRY` + combined judge call; `ground_truth` and `knowledge_base` metrics, applied per-row by which required inputs the row carries), `score.py`, `embeddings.py`, `cost.py`, `langfuse.py`, `merge.py`, `processing.py`, `cron.py` - `core/batch/` — shared provider batch clients: `openai.py`, `gemini.py`, `anthropic.py`, `polling.py`, `operations.py` ## Async @@ -38,4 +38,4 @@ v2 judge fields on `EvaluationRun`: `is_judge_run` (bool marker gating native ju ## Gotchas - Eval traffic deliberately bypasses `/llm/call` (separate code path from production). - Scores sync to Langfuse; durable per-row maps on `evaluation_run` are the resync source of truth. -- v2 (`is_judge_run`) is fully Kaapi-native: no Langfuse dataset/trace/score sync; the native ground-truth judge is one combined `responses.create` per row, structured via `crud/evaluations/judge.py` `METRIC_REGISTRY` so knowledge_base/prompt metrics + verdict slot in later. v1 path is unchanged. +- v2 (`is_judge_run`) is fully Kaapi-native: no Langfuse dataset/trace/score sync; one combined `responses.create` per row scores every applicable metric, structured via `crud/evaluations/judge.py` `METRIC_REGISTRY`. Live metrics: `ground_truth` (backed by `per_item_ground_truth`) and `knowledge_base` (groundedness; no backup column — score + reasoning land in `score_trace_url`) — the latter judges the file_search chunks captured during response generation and only applies to rows that retrieved chunks; `prompt` + weighted verdict slot in later. v1 path is unchanged. From e440002784b71aadfbb8b39f84ad94bcdd578988 Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Tue, 21 Jul 2026 21:06:09 +0530 Subject: [PATCH 2/2] cleanups --- backend/app/crud/evaluations/fast.py | 70 ++++- backend/app/models/response.py | 1 + backend/app/services/response/response.py | 6 +- .../tests/crud/evaluations/test_fast_judge.py | 253 +++++++++++++++++- .../response/test_generate_response.py | 37 ++- 5 files changed, 346 insertions(+), 21 deletions(-) diff --git a/backend/app/crud/evaluations/fast.py b/backend/app/crud/evaluations/fast.py index 33079a85f..5286f1338 100644 --- a/backend/app/crud/evaluations/fast.py +++ b/backend/app/crud/evaluations/fast.py @@ -53,6 +53,7 @@ ) from app.crud.evaluations.judge import ( JudgeInputEnum, + JudgeMetricEnum, JudgeMetricSpec, JudgeResult, build_judge_params, @@ -103,6 +104,21 @@ UNSCOREABLE_EMPTY_GROUND_TRUTH = "empty_ground_truth" UNSCOREABLE_EMBEDDING_FAILED = "embedding_failed" +# How many top KB matches to name in the knowledge_base trace comment. +_KB_TOP_CHUNKS = 3 + + +def _format_top_kb_matches(sorted_chunks: list[dict[str, Any]]) -> str: + """Top-N retrieved chunks as 'biu-1.pdf (90.6%), faq.pdf (66.3%)'. + + Expects chunks pre-sorted by score desc; old S3 payloads may lack filename. + """ + matches = [ + f"{c.get('filename') or 'unknown'} ({c.get('score', 0) * 100:.1f}%)" + for c in sorted_chunks + ] + return ", ".join(matches[:_KB_TOP_CHUNKS]) + # Per-call retry policy for Stage 1 / Stage 2. _RETRY_MAX_ATTEMPTS = 3 @@ -260,7 +276,7 @@ def _responses_call_for_item( failed=False, # Plain dicts (not FileResultChunk) so the unit stays JSON-serializable for S3. retrieved_chunks=[ - {"score": c.score, "text": c.text} + {"score": c.score, "text": c.text, "filename": c.filename} for c in get_file_search_results(response) ], ) @@ -501,6 +517,8 @@ def run_response_chunk( ) # Ask OpenAI to return the file_search hits so knowledge_base can judge them. + # tool_choice stays at the model default (auto) — consistent with normal calls; + # a row where the model doesn't query the KB is scored N/A, not forced to search. if any(t.get("type") == "file_search" for t in base_params.get("tools", [])): base_params["include"] = ["file_search_call.results"] @@ -1118,18 +1136,48 @@ def _stage3_score_and_trace( judge_result = judge_results.get(item_id) if judge_result is not None: + sorted_chunks = sorted( + response.get("retrieved_chunks") or [], + key=lambda c: c.get("score", 0), + reverse=True, + ) + top_matches = _format_top_kb_matches(sorted_chunks) for spec in enabled_metric_specs(): metric_score = judge_result.metrics.get(spec.key) - if metric_score is None: - continue - trace_scores.append( - { - "name": spec.score_name, - "value": round(metric_score.score, 2), - "data_type": "NUMERIC", - "comment": metric_score.reasoning, - } - ) + is_kb = spec.key == JudgeMetricEnum.KNOWLEDGE_BASE + if metric_score is not None: + comment = metric_score.reasoning + if is_kb: + comment = f"{comment} | Top matches: {top_matches}" + trace_scores.append( + { + "name": spec.score_name, + "value": round(metric_score.score, 2), + "data_type": "NUMERIC", + "comment": comment, + } + ) + elif is_kb: + # KB dropped for this row: surface a human reason instead of a bare + # N/A. Placeholder lives only in trace_scores, never the summary avg. + if not sorted_chunks: + # ponytail: empty chunks under auto tool_choice ~= not queried; + # a "was queried" flag would disambiguate an empty-store hit, not + # worth plumbing. + reason = "Knowledge base not queried." + else: + # Chunks present but the judge returned no KB score (rare: a + # well-formed reply that omitted the metric). + reason = "Knowledge base score unavailable for this row." + trace_scores.append( + { + "name": spec.score_name, + "value": "N/A", + "data_type": "CATEGORICAL", + "comment": reason, + "unscoreable": True, + } + ) traces.append( { diff --git a/backend/app/models/response.py b/backend/app/models/response.py index 4de982de6..48af24ce1 100644 --- a/backend/app/models/response.py +++ b/backend/app/models/response.py @@ -42,6 +42,7 @@ class Diagnostics(SQLModel): class FileResultChunk(SQLModel): score: float text: str + filename: str | None = None class CallbackResponse(SQLModel): diff --git a/backend/app/services/response/response.py b/backend/app/services/response/response.py index 808e754f6..bbb322cb1 100644 --- a/backend/app/services/response/response.py +++ b/backend/app/services/response/response.py @@ -45,7 +45,11 @@ def get_file_search_results(response: Response) -> list[FileResultChunk]: for tool_call in response.output: if tool_call.type == "file_search_call": results.extend( - FileResultChunk(score=hit.score, text=hit.text) + FileResultChunk( + score=hit.score, + text=hit.text, + filename=getattr(hit, "filename", None), + ) for hit in tool_call.results ) return results diff --git a/backend/app/tests/crud/evaluations/test_fast_judge.py b/backend/app/tests/crud/evaluations/test_fast_judge.py index c49066b54..804cf4ef8 100644 --- a/backend/app/tests/crud/evaluations/test_fast_judge.py +++ b/backend/app/tests/crud/evaluations/test_fast_judge.py @@ -30,8 +30,10 @@ CHUNK_CONFIG_INDEX, CHUNK_CONFIG_RUN_ID, JOB_TYPE_EVALUATION_FAST_CHUNK, + _format_top_kb_matches, _responses_call_for_item, run_fast_evaluation, + run_response_chunk, ) from app.crud.evaluations.score import ( GROUND_TRUTH_SCORE_NAME, @@ -41,7 +43,7 @@ from app.models import Config, EvaluationDataset, EvaluationRun from app.models.batch_job import BatchJob from app.models.evaluation import RunModeEnum -from app.models.llm.request import ConfigBlob, KaapiCompletionConfig +from app.models.llm.request import ConfigBlob, KaapiCompletionConfig, TextLLMParams from app.models.response import FileResultChunk from app.tests.utils.auth import TestAuthContext from app.tests.utils.test_data import ( @@ -495,7 +497,7 @@ def test_success_with_chunks_returns_serializable_plain_dicts(self): with patch( "app.crud.evaluations.fast.get_file_search_results", return_value=[ - FileResultChunk(score=0.91, text="chunk A"), + FileResultChunk(score=0.91, text="chunk A", filename="doc.pdf"), FileResultChunk(score=0.42, text="chunk B"), ], ): @@ -506,9 +508,10 @@ def test_success_with_chunks_returns_serializable_plain_dicts(self): ) assert result["failed"] is False + # filename flows into the persisted unit so knowledge_base can name its matches. assert result["retrieved_chunks"] == [ - {"score": 0.91, "text": "chunk A"}, - {"score": 0.42, "text": "chunk B"}, + {"score": 0.91, "text": "chunk A", "filename": "doc.pdf"}, + {"score": 0.42, "text": "chunk B", "filename": None}, ] json.dumps(result) # the S3 unit must stay JSON-serializable @@ -543,11 +546,12 @@ def test_error_path_has_no_chunks(self): class TestKnowledgeBaseScoring: - def test_groundedness_scores_only_chunked_rows( + def test_groundedness_scores_chunked_row_and_flags_chunkless_row_na( self, db: Session, user_api_key: TestAuthContext, _s3_store ): - """A chunk-less row is unscoreable for groundedness (absent, not 0), while - ground truth still scores every judged row.""" + """A chunked row gets a numeric KB score; a chunk-less-but-judged row gets an + N/A KB placeholder (not a numeric score, not absent), while ground truth still + scores every judged row.""" eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True) chunked = _resp_result("item-chunked", "Q1", "golden-1") chunked["retrieved_chunks"] = [{"score": 0.9, "text": "supporting chunk"}] @@ -578,5 +582,238 @@ def _judge(params): kb_chunked = _score_named(traces["item-chunked"], KNOWLEDGE_BASE_SCORE_NAME) assert kb_chunked is not None assert kb_chunked["value"] == 0.6 - assert _score_named(traces["item-plain"], KNOWLEDGE_BASE_SCORE_NAME) is None + + # The judged-but-chunkless row surfaces a human N/A placeholder that stays out + # of the summary avg (it is not a numeric 0). + kb_plain = _score_named(traces["item-plain"], KNOWLEDGE_BASE_SCORE_NAME) + assert kb_plain["value"] == "N/A" + assert kb_plain["data_type"] == "CATEGORICAL" + assert kb_plain["unscoreable"] is True + assert kb_plain["comment"] == "Knowledge base not queried." assert _score_named(traces["item-plain"], GROUND_TRUTH_SCORE_NAME) is not None + + def test_kb_scored_row_comment_names_top_matches( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + """A scored KB row appends the top retrieved filenames to its comment.""" + eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True) + chunked = _resp_result("item-1", "Q1", "golden-1") + chunked["retrieved_chunks"] = [ + {"score": 0.906, "text": "supporting chunk", "filename": "biu-1.pdf"}, + {"score": 0.42, "text": "weak chunk", "filename": "faq.pdf"}, + ] + _seed_chunk(db=db, eval_run=eval_run, results=[chunked], store=_s3_store) + + def _judge(params): + if KNOWLEDGE_BASE_SCORE_NAME in params["instructions"]: + return _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.8, "reasoning": "gt"}, + "knowledge_base": {"score": 0.7, "reasoning": "grounded"}, + } + ) + ) + return _judge_response(0.9, "gt only") + + result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) + + kb = _score_named(_trace_by_ref(result)["item-1"], KNOWLEDGE_BASE_SCORE_NAME) + assert kb["data_type"] == "NUMERIC" + assert kb["value"] == 0.7 + # No relevance gate: every retrieved chunk names a match, low scores included. + assert ( + kb["comment"] + == "grounded | Top matches: biu-1.pdf (90.6%), faq.pdf (42.0%)" + ) + + def test_kb_scored_even_when_all_chunks_low_score( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + """Gate removed: any retrieved chunk is judged, however weak its score.""" + eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True) + row = _resp_result("item-1", "Q1", "golden-1") + row["retrieved_chunks"] = [ + {"score": 0.55, "text": "weak but relevant", "filename": "a.pdf"}, + {"score": 0.30, "text": "weaker", "filename": "b.pdf"}, + ] + _seed_chunk(db=db, eval_run=eval_run, results=[row], store=_s3_store) + + def _judge(params): + if KNOWLEDGE_BASE_SCORE_NAME in params["instructions"]: + return _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.9, "reasoning": "gt"}, + "knowledge_base": {"score": 0.6, "reasoning": "partial"}, + } + ) + ) + return _judge_response(0.9, "gt only") + + result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) + + kb = _score_named(_trace_by_ref(result)["item-1"], KNOWLEDGE_BASE_SCORE_NAME) + assert kb["data_type"] == "NUMERIC" + assert kb["value"] == 0.6 + assert kb["comment"] == "partial | Top matches: a.pdf (55.0%), b.pdf (30.0%)" + + def test_kb_na_placeholder_stays_out_of_summary_avg( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + """The KB summary avg reflects only genuinely-scored rows, never the N/A rows.""" + eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True) + scored = _resp_result("item-scored", "Q1", "golden-1") + scored["retrieved_chunks"] = [ + {"score": 0.9, "text": "supporting", "filename": "kb.pdf"} + ] + dropped = _resp_result("item-dropped", "Q2", "golden-2") # no chunks → N/A + _seed_chunk( + db=db, eval_run=eval_run, results=[scored, dropped], store=_s3_store + ) + + def _judge(params): + if KNOWLEDGE_BASE_SCORE_NAME in params["instructions"]: + return _raw_judge_response( + json.dumps( + { + "ground_truth": {"score": 0.8, "reasoning": "gt"}, + "knowledge_base": {"score": 0.6, "reasoning": "grounded"}, + } + ) + ) + return _judge_response(0.9, "gt only") + + result, _ = _run_pipeline(db=db, eval_run=eval_run, judge_side_effect=_judge) + + kb_summary = next( + s + for s in result.score["summary_scores"] + if s["name"] == KNOWLEDGE_BASE_SCORE_NAME + ) + # Only item-scored (0.6) is a real KB score; the N/A row never enters the avg. + assert kb_summary["avg"] == 0.6 + assert kb_summary["total_pairs"] == 1 + + def test_non_kb_metric_none_is_skipped_not_placeholdered( + self, db: Session, user_api_key: TestAuthContext, _s3_store + ): + """A missing non-KB metric (ground_truth) yields no trace score at all — the + N/A placeholder is a KB-only affordance.""" + eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True) + row = _resp_result("item-1", "Q1", "golden-1") + row["retrieved_chunks"] = [ + {"score": 0.9, "text": "supporting", "filename": "kb.pdf"} + ] + _seed_chunk(db=db, eval_run=eval_run, results=[row], store=_s3_store) + + # Judge returns knowledge_base only; ground_truth is silently dropped in parsing. + result, _ = _run_pipeline( + db=db, + eval_run=eval_run, + judge_side_effect=lambda _p: _raw_judge_response( + json.dumps({"knowledge_base": {"score": 0.7, "reasoning": "grounded"}}) + ), + ) + + trace = _trace_by_ref(result)["item-1"] + assert _score_named(trace, KNOWLEDGE_BASE_SCORE_NAME) is not None + # No ground_truth score is emitted — not even an N/A placeholder. + assert _score_named(trace, GROUND_TRUTH_SCORE_NAME) is None + + +class TestFormatTopKbMatches: + """`_format_top_kb_matches` — the human 'Top matches: ...' string for KB comments.""" + + def test_formats_filename_and_percent_to_one_decimal(self) -> None: + result = _format_top_kb_matches( + [ + {"filename": "biu-1.pdf", "score": 0.906}, + {"filename": "faq.pdf", "score": 0.663}, + ] + ) + assert result == "biu-1.pdf (90.6%), faq.pdf (66.3%)" + + def test_includes_all_chunks_regardless_of_score(self) -> None: + result = _format_top_kb_matches( + [ + {"filename": "hi.pdf", "score": 0.9}, + {"filename": "lo.pdf", "score": 0.5}, + ] + ) + assert result == "hi.pdf (90.0%), lo.pdf (50.0%)" + + def test_caps_at_three_matches(self) -> None: + chunks = [{"filename": f"f{i}.pdf", "score": 0.9 - i * 0.01} for i in range(5)] + result = _format_top_kb_matches(chunks) + assert result == "f0.pdf (90.0%), f1.pdf (89.0%), f2.pdf (88.0%)" + + def test_missing_filename_renders_unknown(self) -> None: + assert _format_top_kb_matches([{"score": 0.9}]) == "unknown (90.0%)" + assert ( + _format_top_kb_matches([{"filename": None, "score": 0.8}]) + == "unknown (80.0%)" + ) + + def test_empty_input_is_empty_string(self) -> None: + assert _format_top_kb_matches([]) == "" + + +class TestFileSearchIncludeParam: + """`run_response_chunk` requests file_search hits only when a file_search tool is present, + and never overrides tool_choice (stays at the model default / auto).""" + + def _run_and_capture_base_params( + self, *, db: Session, eval_run: EvaluationRun, tools: list[dict[str, Any]] + ) -> dict[str, Any]: + captured: dict[str, Any] = {} + + def _fake_call(*, openai_client, base_params, item): + captured.update(base_params) + return {"item_id": item["id"], "failed": False, "usage": {}} + + with ( + patch( + "app.crud.evaluations.fast.map_kaapi_to_openai_params", + return_value=({"model": "gpt-4o", "tools": tools}, []), + ), + patch( + "app.crud.evaluations.fast._responses_call_for_item", + side_effect=_fake_call, + ), + patch( + "app.crud.evaluations.fast._upload_unit_to_s3", + return_value="s3://bucket/chunk.json", + ), + ): + run_response_chunk( + session=db, + openai_client=MagicMock(), + eval_run=eval_run, + config=TextLLMParams(model="gpt-4o"), + dataset_items_slice=[{"id": "item-1"}], + chunk_index=0, + log_prefix="[test]", + ) + return captured + + def test_file_search_present_sets_include_and_leaves_tool_choice_default( + self, db: Session, user_api_key: TestAuthContext + ): + eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True) + base_params = self._run_and_capture_base_params( + db=db, eval_run=eval_run, tools=[{"type": "file_search"}] + ) + assert base_params["include"] == ["file_search_call.results"] + assert "tool_choice" not in base_params + + def test_no_file_search_leaves_include_and_tool_choice_unset( + self, db: Session, user_api_key: TestAuthContext + ): + eval_run = _make_run(db=db, user_api_key=user_api_key, is_judge_run=True) + base_params = self._run_and_capture_base_params( + db=db, eval_run=eval_run, tools=[] + ) + assert "include" not in base_params + assert "tool_choice" not in base_params + assert "include" not in base_params diff --git a/backend/app/tests/services/response/response/test_generate_response.py b/backend/app/tests/services/response/response/test_generate_response.py index 9c26466b0..63a432ca3 100644 --- a/backend/app/tests/services/response/response/test_generate_response.py +++ b/backend/app/tests/services/response/response/test_generate_response.py @@ -1,4 +1,5 @@ import pytest +from types import SimpleNamespace from unittest.mock import MagicMock from openai import OpenAIError @@ -6,7 +7,7 @@ from app.core.langfuse.langfuse import LangfuseTracer from app.models import Assistant, ResponsesAPIRequest -from app.services.response.response import generate_response +from app.services.response.response import generate_response, get_file_search_results @pytest.fixture @@ -68,3 +69,37 @@ def test_generate_response_openai_error(assistant_mock: Assistant): assert response is None assert error is not None assert "API failed" in error + + +def _file_search_call(hits: list[SimpleNamespace]) -> SimpleNamespace: + return SimpleNamespace(type="file_search_call", results=hits) + + +class TestGetFileSearchResults: + """`get_file_search_results` flattens file_search hits and carries an optional filename.""" + + def test_captures_filename_when_present_and_none_when_absent(self) -> None: + # Plain stubs (not MagicMock) so getattr(hit, "filename", None) genuinely + # returns None for the second hit instead of a truthy child mock. + with_name = SimpleNamespace(score=0.9, text="chunk A", filename="doc.pdf") + without_name = SimpleNamespace(score=0.4, text="chunk B") + response = SimpleNamespace( + output=[_file_search_call([with_name, without_name])] + ) + + chunks = get_file_search_results(response) + + assert [(c.score, c.text, c.filename) for c in chunks] == [ + (0.9, "chunk A", "doc.pdf"), + (0.4, "chunk B", None), + ] + + def test_ignores_non_file_search_output_items(self) -> None: + message = SimpleNamespace(type="message") + hit = SimpleNamespace(score=0.7, text="chunk", filename="f.pdf") + response = SimpleNamespace(output=[message, _file_search_call([hit])]) + + chunks = get_file_search_results(response) + + assert len(chunks) == 1 + assert chunks[0].filename == "f.pdf"