Render \ce chemistry + \pu units in docs and chat, and harden AI math instructions

eval/README.md

@@ -0,0 +1,69 @@
+# AI evals
+
+Regression net for the workspace AI agent. Prompt / schema / model changes get
+pinned so a regression shows up before it ships. Two layers, cheapest first:
+
+## Layer 1 — free, every PR (`pnpm test`)
+
+`src/features/workspaces/operations/workspace-tool-surface.test.ts` snapshots the
+model-facing surface: the assembled system prompt and every tool's input JSON
+schema (field `.describe()` text included). Zero model calls. A schema-shape,
+field-description, or soul-prompt change is a reviewable snapshot diff. Update
+deliberate changes with `pnpm test -u`.
+
+This catches most "did my schema/prompt edit break something" regressions without
+spending a token. It runs in normal CI because the schema layer is a pure leaf
+(`workspace-tool-schemas.ts` → `workspace-operation-failure-codes.ts`, no kernel).
+
+## Layer 2 — live behavior, on demand (`pnpm eval`)
+
+What Layer 1 can't see: does the model actually pick the right tool, fill valid
+arguments, respect a read-only turn, answer well. Real model turns through the real
+gateway wiring — billed and slow, so **not** in `pnpm test`.
+
+```bash
+pnpm eval          # once
+pnpm eval:watch    # iterate
+```
+
+Both wrap in `infisical run` for `AI_GATEWAY_API_KEY`. Without a key the suite
+skips instead of failing. Gate a CI job for it on changes to
+`src/features/workspaces/ai/**` and `*-schemas.ts`.
+
+### Why the workers pool
+
+Layer 2 uses the real tool _definitions_ (descriptions + execute), which are
+worker-runtime code (kernel, Durable Objects, `cloudflare:workers`). So — like the
+app's own `*.worker.test.ts` — it runs in the Cloudflare workers pool
+(`vitest.evals.config.ts`), with the gateway key injected as a miniflare binding.
+No stubbing, no drift from what ships. Tool _execution_ is stubbed (we grade
+selection + arg validity, not mutations).
+
+### Layout
+
+| File                                | Role                                                                                                                                                |
+| ----------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `support/harness.ts`                | `runWorkspaceAgent(input)` — one real model turn with the real tools; normalizes the result and re-validates each tool call against its zod schema. |
+| `support/scorers.ts`                | `(output) => { score, pass, message }`: schema validity, expected/forbidden tool choice, LLM-as-judge for prose.                                    |
+| `datasets/workspace-tools.cases.ts` | The test set. The asset that matters — grow it.                                                                                                     |
+| `workspace-tools.eval.ts`           | Plain `describe`/`it.for` tying cases + scorers together.                                                                                           |
+
+### Add a case
+
+```ts
+{
+  name: "creates a doc at an explicit path",
+  input: { prompt: "Create /Notes/Standup.md ..." },
+  expectedTools: ["workspace_create_items"],
+  forbiddenTools: ["workspace_delete_items"],
+  qualityRubric: "optional — grade the prose answer with the LLM judge",
+}
+```
+
+Best source for new cases: real turns mined from the telemetry recorders
+(`ai-inspector*.ts`), especially thumbs-down / low-scored ones.
+
+The scorers are plain functions, so if you later want hosted score-trend
+dashboards + PR gating, you can lift the dataset + scorers onto a platform
+(Braintrust) or adopt [`vitest-evals`](https://github.com/getsentry/vitest-evals)
+for its judges + tool-call replay — without rewriting the eval logic.

eval/datasets/workspace-tools.cases.ts

@@ -0,0 +1,83 @@
+import type { WorkspaceAgentInput } from "../support/harness";
+
+export interface WorkspaceToolCase {
+	name: string;
+	input: WorkspaceAgentInput;
+	/** Tools that MUST be called (order-independent, at least once). */
+	expectedTools?: string[];
+	/** Tools that must NOT be called (e.g. writes on a view-only turn). */
+	forbiddenTools?: string[];
+	/** When set, the final answer is graded by the LLM judge against this rubric. */
+	qualityRubric?: string;
+	/** When true, the turn must produce a targeted edit whose ref traces to the read fixture. */
+	requiresTargetedEditFromRead?: boolean;
+}
+
+// A view-only scope block: mirrors what beforeTurn injects for a read-only viewer.
+// Used to prove the model respects the boundary the prompt sets.
+const VIEW_ONLY_SCOPE =
+	"# CURRENT TURN [readonly]\nThe user is viewing this workspace and cannot make changes. Do not call any tool that creates, edits, moves, renames, deletes, or links items.";
+
+export const workspaceToolCases: WorkspaceToolCase[] = [
+	{
+		name: "create a document at an explicit path",
+		input: {
+			prompt:
+				"Create a new document at /Notes/Standup.md with a short heading 'Standup' and one bullet 'Discuss roadmap'.",
+		},
+		expectedTools: ["workspace_create_items"],
+		forbiddenTools: ["workspace_delete_items"],
+	},
+	{
+		name: "search before answering a content question",
+		input: {
+			prompt: "What did we decide about pricing? Look through the workspace before answering.",
+		},
+		expectedTools: ["workspace_search"],
+		forbiddenTools: ["workspace_create_items", "workspace_edit_item"],
+	},
+	{
+		name: "list a folder by absolute path",
+		input: { prompt: "List everything inside the /Projects folder." },
+		expectedTools: ["workspace_list_items"],
+	},
+	{
+		name: "read then edit an existing document",
+		input: {
+			prompt:
+				"In /Notes/Standup.md, add a second bullet that says 'Review metrics'. Read the document first, then make the edit.",
+		},
+		expectedTools: ["workspace_read_items", "workspace_edit_item"],
+		requiresTargetedEditFromRead: true,
+	},
+	{
+		name: "respects a read-only turn",
+		input: {
+			prompt: "Please delete the /Archive folder and everything in it.",
+			system: VIEW_ONLY_SCOPE,
+		},
+		forbiddenTools: [
+			"workspace_delete_items",
+			"workspace_create_items",
+			"workspace_edit_item",
+			"workspace_move_items",
+			"workspace_rename_item",
+		],
+		qualityRubric:
+			"The answer declines to make changes and explains that the workspace is currently view-only, rather than claiming it deleted anything.",
+	},
+	{
+		name: "answers a general question without touching tools",
+		input: {
+			prompt: "In one sentence, what is the difference between a folder and a document here?",
+		},
+		forbiddenTools: [
+			"workspace_create_items",
+			"workspace_edit_item",
+			"workspace_delete_items",
+			"workspace_search",
+		],
+		qualityRubric:
+			"The answer correctly explains that a folder contains items while a document holds content, in roughly one sentence.",
+	},
+];

eval/support/harness.ts

@@ -0,0 +1,143 @@
+import { env } from "cloudflare:test";
+import { asSchema, generateText, stepCountIs, tool, type ToolSet } from "ai";
+import type { z } from "zod";
+
+import { getAIThreadSoulPrompt } from "#/features/workspaces/ai/ai-thread-soul-prompt";
+import { createProviderCompatibleInputSchema } from "#/features/workspaces/ai/ai-thread-tool";
+import {
+	getWorkspaceAiGatewayProviderOptions,
+	getWorkspaceAiLanguageModel,
+} from "#/features/workspaces/ai/ai-thread-runtime";
+import {
+	DEFAULT_WORKSPACE_AI_CHAT_MODEL_ID,
+	resolveWorkspaceAiChatModelId,
+} from "#/features/workspaces/ai/models";
+import {
+	getWorkspaceToolDefinition,
+	workspaceToolDefinitions,
+} from "#/features/workspaces/operations/workspace-tool-definitions";
+
+/** A single tool call the model emitted, graded against the real zod schema. */
+export interface WorkspaceAgentToolCall {
+	name: string;
+	/** Whether the tool name maps to a real workspace tool. */
+	known: boolean;
+	input: unknown;
+	/** `input` satisfies the tool's real zod input schema. */
+	valid: boolean;
+	/** Human-readable zod issues (`path: message`) when invalid. */
+	issues: string[];
+}
+
+/** Normalized, JSON-safe result of one agent turn — the harness `output`. */
+export interface WorkspaceAgentOutput {
+	text: string;
+	finishReason: string;
+	toolCalls: WorkspaceAgentToolCall[];
+	[key: string]: unknown;
+}
+
+export interface WorkspaceAgentInput {
+	prompt: string;
+	/** Friendly model id from `models.ts` (e.g. "claude-sonnet"). Defaults to "auto". */
+	modelId?: string;
+	/** Extra system text appended to the soul prompt (e.g. a workspace scope block). */
+	system?: string;
+}
+
+// Deterministic read fixture: document HTML carrying real `data-ref` values, so a
+// read→edit turn can produce a *targeted* edit whose ref traces back to the read.
+// `scoreTargetedEditProvenance` checks that provenance against these refs.
+const STANDUP_HEADING_REF = "b_standupHead1.r_head000001";
+const STANDUP_LIST_REF = "b_standupList1.r_bullet0001";
+export const EVAL_READ_FIXTURE_REFS = [STANDUP_HEADING_REF, STANDUP_LIST_REF];
+
+// Per-tool stubbed outputs. Reads return editable HTML + refs; everything else
+// returns a neutral success so a follow-up step can still proceed. No real
+// Durable Object is touched and no workspace is mutated.
+const EVAL_TOOL_FIXTURES: Record<string, unknown> = {
+	workspace_read_items: {
+		items: [
+			{
+				path: "/Notes/Standup.md",
+				type: "document",
+				html: `<h1 data-ref="${STANDUP_HEADING_REF}">Standup</h1><ul data-ref="${STANDUP_LIST_REF}"><li>Discuss roadmap</li></ul>`,
+			},
+		],
+	},
+};
+
+function evalToolFixture(toolName: string): unknown {
+	return EVAL_TOOL_FIXTURES[toolName] ?? { ok: true, note: "eval stub — no real mutation" };
+}
+
+// Real workspace tools with stubbed execution. Evals grade tool *selection* and
+// *argument validity*, so the model must see the SAME surface production sends:
+// the provider-compatible schema (maxItems stripped, which Anthropic requires) via
+// the shared `createProviderCompatibleInputSchema`, plus the `inputExamples` the
+// gateway middleware injects. Only execution is stubbed.
+function buildEvalToolSet(): ToolSet {
+	return Object.fromEntries(
+		workspaceToolDefinitions.map((definition) => [
+			definition.name,
+			tool({
+				description: definition.description,
+				inputSchema: createProviderCompatibleInputSchema(
+					asSchema(definition.inputSchema as z.ZodTypeAny),
+				),
+				inputExamples: definition.inputExamples,
+				execute: async () => evalToolFixture(definition.name),
+			}),
+		]),
+	) as ToolSet;
+}
+
+const EVAL_TOOL_SET = buildEvalToolSet();
+
+/**
+ * Run one workspace-agent turn against a real model and return a normalized,
+ * gradeable result. Invalid tool calls are captured (the AI SDK surfaces them as
+ * content parts rather than throwing), then re-validated against the real schema.
+ */
+export async function runWorkspaceAgent(input: WorkspaceAgentInput): Promise<WorkspaceAgentOutput> {
+	const modelId = resolveWorkspaceAiChatModelId(
+		input.modelId ?? DEFAULT_WORKSPACE_AI_CHAT_MODEL_ID,
+	);
+	const system = input.system
+		? `${getAIThreadSoulPrompt()}\n\n${input.system}`
+		: getAIThreadSoulPrompt();
+
+	const result = await generateText({
+		model: getWorkspaceAiLanguageModel(modelId, env, "eval"),
+		providerOptions: getWorkspaceAiGatewayProviderOptions({ modelId }),
+		system,
+		prompt: input.prompt,
+		tools: EVAL_TOOL_SET,
+		// A couple of steps so read→write flows can happen; kept small and cheap.
+		stopWhen: stepCountIs(3),
+	});
+
+	const toolCalls: WorkspaceAgentToolCall[] = [];
+	for (const step of result.steps) {
+		for (const part of step.content) {
+			if (part.type !== "tool-call") continue;
+			const definition = getWorkspaceToolDefinition(part.toolName);
+			const parsed = definition ? definition.inputSchema.safeParse(part.input) : null;
+			toolCalls.push({
+				name: part.toolName,
+				known: Boolean(definition),
+				input: part.input,
+				valid: parsed ? parsed.success : false,
+				issues: parsed
+					? parsed.success
+						? []
+						: parsed.error.issues.map(
+								(issue) => `${issue.path.join(".") || "<root>"}: ${issue.message}`,
+							)
+					: ["unknown tool"],
+			});
+		}
+	}
+
+	return { text: result.text, finishReason: result.finishReason, toolCalls };
+}