feat(metrics): Adherence To Prompt

backend/app/alembic/versions/075_add_judge_columns_to_evaluation_run.py

@@ -1,23 +1,15 @@
-"""Add native LLM-as-judge columns to evaluation_run
+"""Add is_judge_run to evaluation_run
 
 Revision ID: 075
 Revises: 074
 Create Date: 2026-07-16 00:00:00.000000
 
-The v2 judged fast-eval run stores its judge state on the existing evaluation_run
-row rather than a new table. The chunked aggregate task only knows an eval_run_id,
-so the judge intent (is_judge_run) must be durable on the row for the aggregate to
-read at judge time. per_item_ground_truth mirrors per_item_scores as Kaapi's native
-per-row store (v2 never syncs to Langfuse). Judging is system-config only — always
-the fallback model + built-in prompt — so no per-run judge config is persisted.
-
-Both columns are nullable with default NULL: v1 and pre-feature runs carry no judge
-data, so no backfill is needed.
+The chunked aggregate task only knows an eval_run_id, so the judge intent has to be
+durable on the run row for it to read at judge time.
 """
 
 import sqlalchemy as sa
 from alembic import op
-from sqlalchemy.dialects.postgresql import JSONB
 
 revision = "075"
 down_revision = "074"
@@ -38,20 +30,7 @@ def upgrade():
             ),
         ),
     )
-    op.add_column(
-        "evaluation_run",
-        sa.Column(
-            "per_item_ground_truth",
-            JSONB(),
-            nullable=True,
-            comment=(
-                "Durable {ref: score} map of the Adherence to Ground Truth judge "
-                "scores (ref = trace_id when traced, else item_id); Kaapi's own store"
-            ),
-        ),
-    )
 
 
 def downgrade():
-    op.drop_column("evaluation_run", "per_item_ground_truth")
     op.drop_column("evaluation_run", "is_judge_run")

backend/app/api/docs/evaluation/create_evaluation_v2.md

@@ -3,14 +3,18 @@ body with Kaapi's native LLM-as-Judge built in — v1 is left unchanged.
 
 v2 runs are **always fast** and always judged (there is no `run_mode`; batch is
 deferred to a later phase). Every scoreable row is automatically judged (no opt-in
-flag) on **Adherence to Ground Truth**: an LLM judge scores whether the answer
-conveys the same correct information as the dataset's golden answer (0–1, with
-reasoning). Scores, the durable per-row map (`per_item_ground_truth`), and the
-`ground_truth_judge` cost stage are stored natively by Kaapi. v2 runs compute **no
-cosine similarity** and do **not** touch Langfuse.
+flag) by one combined LLM-judge call scoring each applicable metric in [0, 1] with
+reasoning: **Adherence to Ground Truth** (answer conveys the same correct
+information as the golden answer), **Adherence to Prompt** (answer obeys the
+assistant's configured instructions; applies only when the run resolves a config
+prompt), and **Adherence to Knowledge Base** (groundedness of the answer against
+the retrieved chunks; applies only to rows that retrieved chunks). Per-row scores +
+reasoning are stored natively by Kaapi in the `score_trace_url` trace unit; the
+single `judge` cost stage covers the one combined call. v2 runs compute **no cosine
+similarity** and do **not** touch Langfuse.
 
 Judging is system-config only: the judge always uses the configured model
-(`EVAL_JUDGE_MODEL`, default `gpt-5-mini`) and the built-in ground-truth prompt.
+(`EVAL_JUDGE_MODEL`, default `gpt-5-mini`) and the built-in per-metric prompts.
 There is no per-run or ad-hoc judge configuration.
 
 ## Example

backend/app/api/routes/evaluations/dataset_v2.py

@@ -1,8 +1,7 @@
 """v2 Langfuse-free evaluation dataset upload route.
 
-Replica of the v1 dataset upload with the same multipart shape and response, but
-it creates no Langfuse dataset and stores only the original items — duplication is
-recorded in metadata and applied at run time.
+Same multipart shape and response, but no Langfuse dataset is created and only
+the original items are stored — duplication is metadata, applied at run time.
 """
 
 import logging

backend/app/api/routes/evaluations/evaluation_v2.py

@@ -39,8 +39,7 @@ def evaluate_v2(
 ) -> APIResponse[EvaluationRunPublic]:
     """Start a v2 evaluation run.
 
-    v2 runs are always fast and judged on Adherence to Ground Truth; there is no
-    `run_mode` — batch judging is deferred to a later phase.
+    Always fast and judged; there is no `run_mode` — batch judging is deferred.
     """
 
     eval_run = validate_and_start_judged_evaluation(

backend/app/crud/evaluations/cost.py

@@ -11,13 +11,13 @@
     {
         "response":          {model, input_tokens, output_tokens, total_tokens, cost_usd},
         "embedding":         {model, input_tokens, output_tokens, total_tokens, cost_usd},
-        "ground_truth_judge": {model, input_tokens, output_tokens, total_tokens, cost_usd},
+        "judge":             {model, input_tokens, output_tokens, total_tokens, cost_usd},
         "total_cost_usd": float,
     }
 
-Any stage entry is optional. Embedding entries use output_tokens=0. Judge stages
-(one per metric, keyed by the metric's cost_stage) are priced like the response
-stage from per-row usage.
+Any stage entry is optional. Embedding entries use output_tokens=0. One combined call
+grades every metric, so judge tokens can't be split per metric and form a single stage,
+priced from per-row usage like the response stage.
 """
 
 import logging
@@ -115,8 +115,8 @@ def _build_embedding_cost_entry(
     return _build_cost_entry(session=session, model=model, totals=totals)
 
 
-# Fixed top-level keys on eval_run.cost that are NOT per-metric judge stages;
-# everything else at the top level is a judge stage preserved across partial updates.
+# Everything else at eval_run.cost's top level is a judge stage, preserved across
+# partial updates.
 _NON_JUDGE_COST_KEYS: frozenset[str] = frozenset(
     {"response", "embedding", "total_cost_usd"}
 )
@@ -177,8 +177,7 @@ def attach_cost(
 
     Caller is responsible for persisting `eval_run` afterwards. Any stage's
     previously-computed entry on `eval_run.cost` is preserved when that stage's
-    inputs are not supplied, so partial updates never clobber prior data — including
-    prior per-metric judge stages, which are carried forward untouched.
+    inputs are not supplied, so partial updates never clobber prior data.
     """
     try:
         existing_cost = eval_run.cost or {}