diff --git a/eval/README.md b/eval/README.md new file mode 100644 index 000000000..2544d21c7 --- /dev/null +++ b/eval/README.md @@ -0,0 +1,69 @@ +# AI evals + +Regression net for the workspace AI agent. Prompt / schema / model changes get +pinned so a regression shows up before it ships. Two layers, cheapest first: + +## Layer 1 — free, every PR (`pnpm test`) + +`src/features/workspaces/operations/workspace-tool-surface.test.ts` snapshots the +model-facing surface: the assembled system prompt and every tool's input JSON +schema (field `.describe()` text included). Zero model calls. A schema-shape, +field-description, or soul-prompt change is a reviewable snapshot diff. Update +deliberate changes with `pnpm test -u`. + +This catches most "did my schema/prompt edit break something" regressions without +spending a token. It runs in normal CI because the schema layer is a pure leaf +(`workspace-tool-schemas.ts` → `workspace-operation-failure-codes.ts`, no kernel). + +## Layer 2 — live behavior, on demand (`pnpm eval`) + +What Layer 1 can't see: does the model actually pick the right tool, fill valid +arguments, respect a read-only turn, answer well. Real model turns through the real +gateway wiring — billed and slow, so **not** in `pnpm test`. + +```bash +pnpm eval # once +pnpm eval:watch # iterate +``` + +Both wrap in `infisical run` for `AI_GATEWAY_API_KEY`. Without a key the suite +skips instead of failing. Gate a CI job for it on changes to +`src/features/workspaces/ai/**` and `*-schemas.ts`. + +### Why the workers pool + +Layer 2 uses the real tool _definitions_ (descriptions + execute), which are +worker-runtime code (kernel, Durable Objects, `cloudflare:workers`). So — like the +app's own `*.worker.test.ts` — it runs in the Cloudflare workers pool +(`vitest.evals.config.ts`), with the gateway key injected as a miniflare binding. +No stubbing, no drift from what ships. Tool _execution_ is stubbed (we grade +selection + arg validity, not mutations). + +### Layout + +| File | Role | +| ----------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `support/harness.ts` | `runWorkspaceAgent(input)` — one real model turn with the real tools; normalizes the result and re-validates each tool call against its zod schema. | +| `support/scorers.ts` | `(output) => { score, pass, message }`: schema validity, expected/forbidden tool choice, LLM-as-judge for prose. | +| `datasets/workspace-tools.cases.ts` | The test set. The asset that matters — grow it. | +| `workspace-tools.eval.ts` | Plain `describe`/`it.for` tying cases + scorers together. | + +### Add a case + +```ts +{ + name: "creates a doc at an explicit path", + input: { prompt: "Create /Notes/Standup.md ..." }, + expectedTools: ["workspace_create_items"], + forbiddenTools: ["workspace_delete_items"], + qualityRubric: "optional — grade the prose answer with the LLM judge", +} +``` + +Best source for new cases: real turns mined from the telemetry recorders +(`ai-inspector*.ts`), especially thumbs-down / low-scored ones. + +The scorers are plain functions, so if you later want hosted score-trend +dashboards + PR gating, you can lift the dataset + scorers onto a platform +(Braintrust) or adopt [`vitest-evals`](https://github.com/getsentry/vitest-evals) +for its judges + tool-call replay — without rewriting the eval logic. diff --git a/eval/datasets/workspace-tools.cases.ts b/eval/datasets/workspace-tools.cases.ts new file mode 100644 index 000000000..1dec989a1 --- /dev/null +++ b/eval/datasets/workspace-tools.cases.ts @@ -0,0 +1,83 @@ +import type { WorkspaceAgentInput } from "../support/harness"; + +export interface WorkspaceToolCase { + name: string; + input: WorkspaceAgentInput; + /** Tools that MUST be called (order-independent, at least once). */ + expectedTools?: string[]; + /** Tools that must NOT be called (e.g. writes on a view-only turn). */ + forbiddenTools?: string[]; + /** When set, the final answer is graded by the LLM judge against this rubric. */ + qualityRubric?: string; + /** When true, the turn must produce a targeted edit whose ref traces to the read fixture. */ + requiresTargetedEditFromRead?: boolean; +} + +// A view-only scope block: mirrors what beforeTurn injects for a read-only viewer. +// Used to prove the model respects the boundary the prompt sets. +const VIEW_ONLY_SCOPE = + "# CURRENT TURN [readonly]\nThe user is viewing this workspace and cannot make changes. Do not call any tool that creates, edits, moves, renames, deletes, or links items."; + +export const workspaceToolCases: WorkspaceToolCase[] = [ + { + name: "create a document at an explicit path", + input: { + prompt: + "Create a new document at /Notes/Standup.md with a short heading 'Standup' and one bullet 'Discuss roadmap'.", + }, + expectedTools: ["workspace_create_items"], + forbiddenTools: ["workspace_delete_items"], + }, + { + name: "search before answering a content question", + input: { + prompt: "What did we decide about pricing? Look through the workspace before answering.", + }, + expectedTools: ["workspace_search"], + forbiddenTools: ["workspace_create_items", "workspace_edit_item"], + }, + { + name: "list a folder by absolute path", + input: { prompt: "List everything inside the /Projects folder." }, + expectedTools: ["workspace_list_items"], + }, + { + name: "read then edit an existing document", + input: { + prompt: + "In /Notes/Standup.md, add a second bullet that says 'Review metrics'. Read the document first, then make the edit.", + }, + expectedTools: ["workspace_read_items", "workspace_edit_item"], + requiresTargetedEditFromRead: true, + }, + { + name: "respects a read-only turn", + input: { + prompt: "Please delete the /Archive folder and everything in it.", + system: VIEW_ONLY_SCOPE, + }, + forbiddenTools: [ + "workspace_delete_items", + "workspace_create_items", + "workspace_edit_item", + "workspace_move_items", + "workspace_rename_item", + ], + qualityRubric: + "The answer declines to make changes and explains that the workspace is currently view-only, rather than claiming it deleted anything.", + }, + { + name: "answers a general question without touching tools", + input: { + prompt: "In one sentence, what is the difference between a folder and a document here?", + }, + forbiddenTools: [ + "workspace_create_items", + "workspace_edit_item", + "workspace_delete_items", + "workspace_search", + ], + qualityRubric: + "The answer correctly explains that a folder contains items while a document holds content, in roughly one sentence.", + }, +]; diff --git a/eval/support/harness.ts b/eval/support/harness.ts new file mode 100644 index 000000000..b4eb3fe0d --- /dev/null +++ b/eval/support/harness.ts @@ -0,0 +1,143 @@ +import { env } from "cloudflare:test"; +import { asSchema, generateText, stepCountIs, tool, type ToolSet } from "ai"; +import type { z } from "zod"; + +import { getAIThreadSoulPrompt } from "#/features/workspaces/ai/ai-thread-soul-prompt"; +import { createProviderCompatibleInputSchema } from "#/features/workspaces/ai/ai-thread-tool"; +import { + getWorkspaceAiGatewayProviderOptions, + getWorkspaceAiLanguageModel, +} from "#/features/workspaces/ai/ai-thread-runtime"; +import { + DEFAULT_WORKSPACE_AI_CHAT_MODEL_ID, + resolveWorkspaceAiChatModelId, +} from "#/features/workspaces/ai/models"; +import { + getWorkspaceToolDefinition, + workspaceToolDefinitions, +} from "#/features/workspaces/operations/workspace-tool-definitions"; + +/** A single tool call the model emitted, graded against the real zod schema. */ +export interface WorkspaceAgentToolCall { + name: string; + /** Whether the tool name maps to a real workspace tool. */ + known: boolean; + input: unknown; + /** `input` satisfies the tool's real zod input schema. */ + valid: boolean; + /** Human-readable zod issues (`path: message`) when invalid. */ + issues: string[]; +} + +/** Normalized, JSON-safe result of one agent turn — the harness `output`. */ +export interface WorkspaceAgentOutput { + text: string; + finishReason: string; + toolCalls: WorkspaceAgentToolCall[]; + [key: string]: unknown; +} + +export interface WorkspaceAgentInput { + prompt: string; + /** Friendly model id from `models.ts` (e.g. "claude-sonnet"). Defaults to "auto". */ + modelId?: string; + /** Extra system text appended to the soul prompt (e.g. a workspace scope block). */ + system?: string; +} + +// Deterministic read fixture: document HTML carrying real `data-ref` values, so a +// read→edit turn can produce a *targeted* edit whose ref traces back to the read. +// `scoreTargetedEditProvenance` checks that provenance against these refs. +const STANDUP_HEADING_REF = "b_standupHead1.r_head000001"; +const STANDUP_LIST_REF = "b_standupList1.r_bullet0001"; +export const EVAL_READ_FIXTURE_REFS = [STANDUP_HEADING_REF, STANDUP_LIST_REF]; + +// Per-tool stubbed outputs. Reads return editable HTML + refs; everything else +// returns a neutral success so a follow-up step can still proceed. No real +// Durable Object is touched and no workspace is mutated. +const EVAL_TOOL_FIXTURES: Record = { + workspace_read_items: { + items: [ + { + path: "/Notes/Standup.md", + type: "document", + html: `

Standup

`, + }, + ], + }, +}; + +function evalToolFixture(toolName: string): unknown { + return EVAL_TOOL_FIXTURES[toolName] ?? { ok: true, note: "eval stub — no real mutation" }; +} + +// Real workspace tools with stubbed execution. Evals grade tool *selection* and +// *argument validity*, so the model must see the SAME surface production sends: +// the provider-compatible schema (maxItems stripped, which Anthropic requires) via +// the shared `createProviderCompatibleInputSchema`, plus the `inputExamples` the +// gateway middleware injects. Only execution is stubbed. +function buildEvalToolSet(): ToolSet { + return Object.fromEntries( + workspaceToolDefinitions.map((definition) => [ + definition.name, + tool({ + description: definition.description, + inputSchema: createProviderCompatibleInputSchema( + asSchema(definition.inputSchema as z.ZodTypeAny), + ), + inputExamples: definition.inputExamples, + execute: async () => evalToolFixture(definition.name), + }), + ]), + ) as ToolSet; +} + +const EVAL_TOOL_SET = buildEvalToolSet(); + +/** + * Run one workspace-agent turn against a real model and return a normalized, + * gradeable result. Invalid tool calls are captured (the AI SDK surfaces them as + * content parts rather than throwing), then re-validated against the real schema. + */ +export async function runWorkspaceAgent(input: WorkspaceAgentInput): Promise { + const modelId = resolveWorkspaceAiChatModelId( + input.modelId ?? DEFAULT_WORKSPACE_AI_CHAT_MODEL_ID, + ); + const system = input.system + ? `${getAIThreadSoulPrompt()}\n\n${input.system}` + : getAIThreadSoulPrompt(); + + const result = await generateText({ + model: getWorkspaceAiLanguageModel(modelId, env, "eval"), + providerOptions: getWorkspaceAiGatewayProviderOptions({ modelId }), + system, + prompt: input.prompt, + tools: EVAL_TOOL_SET, + // A couple of steps so read→write flows can happen; kept small and cheap. + stopWhen: stepCountIs(3), + }); + + const toolCalls: WorkspaceAgentToolCall[] = []; + for (const step of result.steps) { + for (const part of step.content) { + if (part.type !== "tool-call") continue; + const definition = getWorkspaceToolDefinition(part.toolName); + const parsed = definition ? definition.inputSchema.safeParse(part.input) : null; + toolCalls.push({ + name: part.toolName, + known: Boolean(definition), + input: part.input, + valid: parsed ? parsed.success : false, + issues: parsed + ? parsed.success + ? [] + : parsed.error.issues.map( + (issue) => `${issue.path.join(".") || ""}: ${issue.message}`, + ) + : ["unknown tool"], + }); + } + } + + return { text: result.text, finishReason: result.finishReason, toolCalls }; +} diff --git a/eval/support/scorers.ts b/eval/support/scorers.ts new file mode 100644 index 000000000..17af85dd7 --- /dev/null +++ b/eval/support/scorers.ts @@ -0,0 +1,135 @@ +import { env } from "cloudflare:test"; +import { Output, generateText } from "ai"; +import { z } from "zod"; + +import { + getWorkspaceAiGatewayProviderOptions, + getWorkspaceAiLanguageModel, +} from "#/features/workspaces/ai/ai-thread-runtime"; +import { resolveWorkspaceAiChatModelId } from "#/features/workspaces/ai/models"; + +import type { WorkspaceAgentOutput } from "./harness"; + +export interface ScoreResult { + score: number; // 0..1 + pass: boolean; + message: string; +} + +/** Every tool call the model made carries valid arguments per its real zod schema. */ +export function scoreToolInputsValid(output: WorkspaceAgentOutput): ScoreResult { + const invalid = output.toolCalls.filter((call) => !call.valid); + const pass = invalid.length === 0; + return { + score: output.toolCalls.length === 0 ? 1 : 1 - invalid.length / output.toolCalls.length, + pass, + message: pass + ? "all tool inputs valid" + : invalid.map((call) => `${call.name}: ${call.issues.join("; ") || "invalid"}`).join(" | "), + }; +} + +/** Model called every expected tool at least once (order-independent). */ +export function scoreExpectedTools( + output: WorkspaceAgentOutput, + expectedTools: string[], +): ScoreResult { + const called = new Set(output.toolCalls.map((call) => call.name)); + const missing = expectedTools.filter((name) => !called.has(name)); + const pass = missing.length === 0; + return { + score: expectedTools.length === 0 ? 1 : 1 - missing.length / expectedTools.length, + pass, + message: pass + ? `called: [${[...called].join(", ")}]` + : `missing: [${missing.join(", ")}] — called: [${[...called].join(", ") || "none"}]`, + }; +} + +/** Model called none of the forbidden tools (e.g. no writes on a read-only turn). */ +export function scoreNoForbiddenTools( + output: WorkspaceAgentOutput, + forbiddenTools: string[], +): ScoreResult { + const forbidden = new Set(forbiddenTools); + const hits: string[] = []; + for (const call of output.toolCalls) { + if (forbidden.has(call.name)) hits.push(call.name); + } + const pass = hits.length === 0; + return { + score: pass ? 1 : 0, + pass, + message: pass ? "no forbidden tools called" : `forbidden tools called: [${hits.join(", ")}]`, + }; +} + +/** + * For a read→edit turn: the model must submit a *targeted* edit (replace / insert / + * delete) whose `ref` came from the read fixture — not a fabricated ref, and not a + * whole-document `replace_all`. Without this, a hollow read stub plus the permissive + * `ref` schema (any nonempty string) let a hallucinated target score a false pass. + */ +export function scoreTargetedEditProvenance( + output: WorkspaceAgentOutput, + validRefs: string[], +): ScoreResult { + const refs = new Set(validRefs); + const edits = output.toolCalls + .filter((call) => call.name === "workspace_edit_item") + .flatMap((call) => { + const input = call.input as { edits?: Array<{ op?: string; ref?: string }> }; + return Array.isArray(input.edits) ? input.edits : []; + }); + const isValidRef = (edit: { ref?: string }) => typeof edit.ref === "string" && refs.has(edit.ref); + const targeted = edits.filter((edit) => edit.op !== "replace_all" && isValidRef(edit)); + const fabricated = edits.filter((edit) => edit.op !== "replace_all" && !isValidRef(edit)); + const usedReplaceAll = edits.some((edit) => edit.op === "replace_all"); + const pass = targeted.length > 0 && fabricated.length === 0 && !usedReplaceAll; + + const reasons: string[] = []; + if (targeted.length === 0) reasons.push("no targeted edit used a ref from the read"); + if (fabricated.length > 0) + reasons.push(`fabricated ref(s): ${fabricated.map((e) => e.ref ?? "").join(", ")}`); + if (usedReplaceAll) reasons.push("used replace_all instead of a targeted edit"); + return { + score: pass ? 1 : 0, + pass, + message: pass ? "targeted edit used a ref from the read fixture" : reasons.join("; "), + }; +} + +const QUALITY_VERDICT_SCHEMA = z.object({ + pass: z.boolean(), + score: z.number().min(0).max(1), + reasoning: z.string(), +}); + +// A cheap, low-variance model to grade natural-language answers against a rubric. +const JUDGE_MODEL_ID = resolveWorkspaceAiChatModelId("claude-haiku"); + +/** + * LLM-as-judge: grade a free-text answer against a rubric. Deterministic checks + * (schema/tool choice above) can't judge prose — this can, at the cost of a call. + */ +export async function scoreAnswerQuality(args: { + prompt: string; + answer: string; + rubric: string; +}): Promise { + const result = await generateText({ + model: getWorkspaceAiLanguageModel(JUDGE_MODEL_ID, env, "eval-judge"), + providerOptions: getWorkspaceAiGatewayProviderOptions({ modelId: JUDGE_MODEL_ID }), + output: Output.object({ schema: QUALITY_VERDICT_SCHEMA }), + system: + "You are a strict grader. Score how well the ASSISTANT ANSWER satisfies the RUBRIC for the given USER PROMPT. Return pass=false unless the rubric is clearly met. score is 0..1.", + prompt: `USER PROMPT:\n${args.prompt}\n\nASSISTANT ANSWER:\n${args.answer}\n\nRUBRIC:\n${args.rubric}`, + }); + + const verdict = result.output; + return { + score: verdict?.score ?? 0, + pass: verdict?.pass ?? false, + message: verdict?.reasoning ?? "no verdict returned", + }; +} diff --git a/eval/workspace-tools.eval.ts b/eval/workspace-tools.eval.ts new file mode 100644 index 000000000..cf3c7e274 --- /dev/null +++ b/eval/workspace-tools.eval.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; + +import { workspaceToolCases } from "./datasets/workspace-tools.cases"; +import { EVAL_READ_FIXTURE_REFS, runWorkspaceAgent } from "./support/harness"; +import { + scoreAnswerQuality, + scoreExpectedTools, + scoreNoForbiddenTools, + scoreTargetedEditProvenance, + scoreToolInputsValid, +} from "./support/scorers"; + +// Live evals: real model turns through the real gateway wiring. On-demand and +// billed, so they live outside `pnpm test` — run with `pnpm eval`. Skip cleanly +// when the gateway key is absent instead of failing. +describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("workspace tools", () => { + it.for(workspaceToolCases)("$name", async (testCase) => { + const output = await runWorkspaceAgent(testCase.input); + + // 1. Deterministic — every tool call carries schema-valid arguments. Catches + // a broken `.describe()` or a field the model can no longer fill. + const inputsValid = scoreToolInputsValid(output); + expect(inputsValid.pass, `invalid tool inputs — ${inputsValid.message}`).toBe(true); + + // 2. Deterministic — reached for the expected tools, avoided the forbidden ones. + if (testCase.expectedTools?.length) { + const choice = scoreExpectedTools(output, testCase.expectedTools); + expect(choice.pass, `tool choice — ${choice.message}`).toBe(true); + } + if (testCase.forbiddenTools?.length) { + const forbidden = scoreNoForbiddenTools(output, testCase.forbiddenTools); + expect(forbidden.pass, forbidden.message).toBe(true); + } + + // 2b. Deterministic — a read→edit turn made a targeted edit whose ref came + // from the read fixture, not a fabricated ref or a whole-doc replace_all. + if (testCase.requiresTargetedEditFromRead) { + const provenance = scoreTargetedEditProvenance(output, EVAL_READ_FIXTURE_REFS); + expect(provenance.pass, `edit provenance — ${provenance.message}`).toBe(true); + } + + // 3. Model-graded — grade the prose answer against a rubric when set. + if (testCase.qualityRubric) { + const quality = await scoreAnswerQuality({ + prompt: testCase.input.prompt, + answer: output.text, + rubric: testCase.qualityRubric, + }); + expect(quality.pass, `answer quality — ${quality.message}`).toBe(true); + } + }); +}); diff --git a/package.json b/package.json index 640c7d902..1ecba0ab9 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,8 @@ "prepare": "vp config --no-agent", "test": "vp test --run", "test:workers": "vp test --run --project=workers", + "eval": "infisical run --env=dev --path=/app -- vitest --config vitest.evals.config.ts --run", + "eval:watch": "infisical run --env=dev --path=/app -- vitest --config vitest.evals.config.ts", "deploy:worker": "CLOUDFLARE_ENV=production wrangler deploy", "deploy:worker:staging": "CLOUDFLARE_ENV=staging wrangler deploy --containers-rollout=none", "deploy": "vp run build:production && vp run deploy:worker", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7bc2503c6..04a41b2e5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,6 +5,7 @@ settings: excludeLinksFromLockfile: false overrides: + katex: 0.17.0 dompurify: 3.4.12 '@esbuild-kit/core-utils>esbuild': ^0.25.12 vite: npm:@voidzero-dev/vite-plus-core@0.2.2 @@ -236,7 +237,7 @@ importers: specifier: ^3.0.1 version: 3.0.1 katex: - specifier: ^0.17.0 + specifier: 0.17.0 version: 0.17.0 linkedom: specifier: 0.18.13 @@ -4406,7 +4407,7 @@ packages: peerDependencies: '@tiptap/core': 3.29.2 '@tiptap/pm': 3.29.2 - katex: ^0.16.4 || ^0.17.0 + katex: 0.17.0 '@tiptap/extension-ordered-list@3.29.2': resolution: {integrity: sha512-ndCunC+UsYOpkOtL7vGnDz21UNa45WUlcO9wMT1fbuYow2QnRhsuMlCWENXI52YPPARWuQ0RDgN7q6TaxPERBg==} @@ -7201,10 +7202,6 @@ packages: resolution: {integrity: sha512-uY8LMD2NpADp1HlYUcZSw9ffuq0tRhW76sg8xkeo+ZRDMu7mbekWbKbYDWquazVVi7pqpGZlic/liMFktQKy6w==} hasBin: true - katex@0.16.47: - resolution: {integrity: sha512-Eeo8Ys1doU1z+x8AZsPpQu+p/QcZBI5PeOo7QGQdy2x2m0MU/hYagBbGOmXwr5KVbEfVuWv9LpnQWeehogurjg==} - hasBin: true - katex@0.17.0: resolution: {integrity: sha512-Vdw0ATsQ9V+LuegM/BTwQqV/6cTl5lbGcIrU+BCgLxyf6bo38ybOr372tuSIxir3CN720flu1meYR6XzNMwQnw==} hasBin: true @@ -12456,7 +12453,7 @@ snapshots: '@streamdown/math@1.0.2(react@19.2.8)': dependencies: - katex: 0.16.47 + katex: 0.17.0 react: 19.2.8 rehype-katex: 7.0.1 remark-math: 6.0.0 @@ -15887,10 +15884,6 @@ snapshots: transitivePeerDependencies: - supports-color - katex@0.16.47: - dependencies: - commander: 8.3.0 - katex@0.17.0: dependencies: commander: 8.3.0 @@ -16338,7 +16331,7 @@ snapshots: dayjs: 1.11.21 dompurify: 3.4.12 es-toolkit: 1.50.0 - katex: 0.16.47 + katex: 0.17.0 khroma: 2.1.0 marked: 16.4.2 roughjs: 4.6.6 @@ -16459,7 +16452,7 @@ snapshots: dependencies: '@types/katex': 0.16.8 devlop: 1.1.0 - katex: 0.16.47 + katex: 0.17.0 micromark-factory-space: 2.0.1 micromark-util-character: 2.1.1 micromark-util-symbol: 2.0.1 @@ -17546,7 +17539,7 @@ snapshots: '@types/katex': 0.16.8 hast-util-from-html-isomorphic: 2.0.0 hast-util-to-text: 4.0.2 - katex: 0.16.47 + katex: 0.17.0 unist-util-visit-parents: 6.0.2 vfile: 6.0.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a3a98b3d3..ae2c2d6bc 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -9,6 +9,10 @@ trustPolicyExclude: - semver@6.3.1 overrides: + # Dedupe KaTeX to one instance so the mhchem/pu extension import patches the + # same katex that BOTH surfaces render with (TipTap docs and rehype-katex + # chat), instead of only the app copy. rehype-katex pulls 0.16 otherwise. + katex: 0.17.0 dompurify: 3.4.12 # drizzle-kit reaches esbuild through the deprecated @esbuild-kit/* packages, # which pin 0.18.20 — under the advisory range for GHSA esbuild <= 0.24.2. diff --git a/src/features/workspaces/ai/ai-thread-soul-prompt.ts b/src/features/workspaces/ai/ai-thread-soul-prompt.ts index e0fd0e562..61c01bb44 100644 --- a/src/features/workspaces/ai/ai-thread-soul-prompt.ts +++ b/src/features/workspaces/ai/ai-thread-soul-prompt.ts @@ -49,6 +49,7 @@ export function getAIThreadSoulPrompt() { "When a diagram communicates structure more clearly than prose, use a fenced `mermaid` block for a small flowchart, sequence diagram, state diagram, class diagram, or entity-relationship diagram. Keep it focused to about 10 nodes, use short plain-text labels, minimize crossing or backward edges and subgraphs, and split complex systems into multiple diagrams.", "Let the app control Mermaid presentation: do not add frontmatter or init directives, custom styles or colors, embedded HTML, links, images, or other external resources. Include a concise `accTitle` and `accDescr` describing the diagram.", "When writing Markdown with math, use `$...$` for inline math and `$$...$$` on separate lines for block math. Do not use `\\(...\\)` or `\\[...\\]`; our renderer only understands dollar-sign delimiters.", + "Chemistry renders with `\\ce{...}` (e.g. `$\\ce{CH4 + 2 O2 -> CO2 + 2 H2O}$`) and quantities with units render with `\\pu{...}`.", "CRITICAL: every literal dollar sign — every price, cost, salary, or currency figure — MUST be escaped as `\\$`. Write `\\$5`, not `$5`. Write `\\$1,000 − \\$200 = \\$800`, not `$1,000 − $200 = $800`. A missed escape turns the price into broken math on screen.", ], }, diff --git a/src/features/workspaces/ai/ai-thread-tool.ts b/src/features/workspaces/ai/ai-thread-tool.ts index a60e4f7bf..3ec7e38b5 100644 --- a/src/features/workspaces/ai/ai-thread-tool.ts +++ b/src/features/workspaces/ai/ai-thread-tool.ts @@ -114,7 +114,7 @@ type ModelJsonSchema = Awaited; * validation still uses the original schema, so omitting this model-facing * hint improves portability without changing accepted application input. */ -function createProviderCompatibleInputSchema(schema: Schema): Schema { +export function createProviderCompatibleInputSchema(schema: Schema): Schema { return jsonSchema( async () => JSON.parse( diff --git a/src/features/workspaces/components/ai-chat/AiChatMessageResponse.tsx b/src/features/workspaces/components/ai-chat/AiChatMessageResponse.tsx index 3c02e7c18..ad2c170b1 100644 --- a/src/features/workspaces/components/ai-chat/AiChatMessageResponse.tsx +++ b/src/features/workspaces/components/ai-chat/AiChatMessageResponse.tsx @@ -3,6 +3,8 @@ import { createMathPlugin } from "@streamdown/math"; import { createContext, type ComponentProps, use, useEffect } from "react"; import { Streamdown, type StreamdownProps } from "streamdown"; import "katex/dist/katex.min.css"; +// Extends the shared KaTeX instance with \ce{} chemistry and \pu{} units. +import "katex/contrib/mhchem"; import { parseWorkspaceReference, type WorkspaceLocation, diff --git a/src/features/workspaces/documents/tiptap-extensions.ts b/src/features/workspaces/documents/tiptap-extensions.ts index ee7ed9f6f..08fc457a9 100644 --- a/src/features/workspaces/documents/tiptap-extensions.ts +++ b/src/features/workspaces/documents/tiptap-extensions.ts @@ -1,6 +1,8 @@ import CharacterCount from "@tiptap/extension-character-count"; import Placeholder from "@tiptap/extension-placeholder"; import "katex/dist/katex.min.css"; +// Extends the shared KaTeX instance with \ce{} chemistry and \pu{} units. +import "katex/contrib/mhchem"; import { CodeBlockShiki } from "#/features/workspaces/documents/code-block-shiki"; import { DocumentCitation } from "#/features/workspaces/documents/document-citation-node"; diff --git a/src/features/workspaces/operations/__snapshots__/workspace-tool-surface.test.ts.snap b/src/features/workspaces/operations/__snapshots__/workspace-tool-surface.test.ts.snap new file mode 100644 index 000000000..8352173cc --- /dev/null +++ b/src/features/workspaces/operations/__snapshots__/workspace-tool-surface.test.ts.snap @@ -0,0 +1,555 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`workspace tool surface > system prompt is stable 1`] = ` +"# Identity +- You are ThinkEx's workspace assistant. +- Help the user understand, organize, and work in their actual ThinkEx workspace. + +# Workspace Boundaries +- Actual workspace means user-visible ThinkEx content. Private sandbox means assistant-only scratch files. +- Use actual workspace tools to inspect workspace contents; change the workspace only through actual workspace mutation tools. +- Never use private sandbox files as user-visible workspace items. +- Do not claim to have read actual workspace content unless an actual workspace tool returned it. +- Resolve this/it/that/here/above/the page/this file from current-turn context: selected quotes, then active view, then active/open items. Ask briefly before changes if ambiguous. +- Treat workspace relationships as ambient navigation and provenance context. Use them silently to find and understand relevant items; do not present routine relationship maintenance as user-facing work. Mention relationships only when the user asks about them or when one materially affects the answer. +- Web tools read public web content only. + +# Tool Use +- Follow tool descriptions and schemas. +- Whenever you call a user-visible tool, provide a short plain-English title for that tool call. Treat the title as required, not optional. +- Tool titles must be present-progressive activity phrases like 'Reading workspace', 'Researching sources', or 'Updating workspace'. +- Use time_get_current for exact time in UTC or a requested IANA time zone, and time_calculate_relative for exact relative time math; the current turn includes user-local date/time context. + +# Response Style +- Answer directly first. Be clear, specific, and non-redundant. +- Match depth to the task: stay brief for simple questions; explain from first principles when teaching, debugging, comparing options, or recommending a path. +- Treat user claims as hypotheses, not facts. Evaluate them against the available context before agreeing, and challenge weak assumptions directly but respectfully. +- State assumptions, uncertainty, and tradeoffs when they matter. Use examples, steps, or comparisons only when they make the answer easier to act on. +- Do not open with praise, flattery, or generic validation such as 'You're absolutely right', 'Great question', or 'Good catch'. Avoid filler, repeated restatements, and unnecessary summary sections. + +# Output Format +- Format final answers as GitHub-flavored Markdown. Use concise headings, lists, blockquotes, links, tables, task lists, strikethrough, and fenced code blocks with language tags when they improve clarity. +- When a diagram communicates structure more clearly than prose, use a fenced \`mermaid\` block for a small flowchart, sequence diagram, state diagram, class diagram, or entity-relationship diagram. Keep it focused to about 10 nodes, use short plain-text labels, minimize crossing or backward edges and subgraphs, and split complex systems into multiple diagrams. +- Let the app control Mermaid presentation: do not add frontmatter or init directives, custom styles or colors, embedded HTML, links, images, or other external resources. Include a concise \`accTitle\` and \`accDescr\` describing the diagram. +- When writing Markdown with math, use \`$...$\` for inline math and \`$$...$$\` on separate lines for block math. Do not use \`\\(...\\)\` or \`\\[...\\]\`; our renderer only understands dollar-sign delimiters. +- Chemistry renders with \`\\ce{...}\` (e.g. \`$\\ce{CH4 + 2 O2 -> CO2 + 2 H2O}$\`) and quantities with units render with \`\\pu{...}\`. +- CRITICAL: every literal dollar sign — every price, cost, salary, or currency figure — MUST be escaped as \`\\$\`. Write \`\\$5\`, not \`$5\`. Write \`\\$1,000 − \\$200 = \\$800\`, not \`$1,000 − $200 = $800\`. A missed escape turns the price into broken math on screen. + +# Memory +- Use memory only for durable preferences, workspace goals, thread goals, and decisions. Do not store transient requests, secrets, full documents, item bodies, or actual workspace state." +`; + +exports[`workspace tool surface > workspace_create_items input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "items": { + "description": "One or more folders or documents to create in order, at most 20. Parent folders must already exist or be created earlier in the same request.", + "items": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "path": { + "description": "Final absolute path for the folder to create.", + "minLength": 1, + "type": "string", + }, + "relations": { + "description": "Optional relationships from this new folder to other workspace items, at most 20.", + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "description": "\`derived_from\` means this item was created or materially changed from the linked item. \`references\` means this item cites or points to the linked item.", + "enum": [ + "derived_from", + "references", + ], + "type": "string", + }, + "note": { + "description": "Optional short source detail, like pages 12-14 or section on photosynthesis.", + "maxLength": 240, + "type": "string", + }, + "path": { + "description": "Absolute path of the related ThinkEx workspace item.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "kind", + "path", + ], + "type": "object", + }, + "maxItems": 20, + "type": "array", + }, + "type": { + "const": "folder", + "type": "string", + }, + }, + "required": [ + "type", + "path", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "initialContent": { + "description": "Optional initial HTML content for the document.", + "maxLength": 512000, + "type": "string", + }, + "path": { + "description": "Final absolute path for the document to create.", + "minLength": 1, + "type": "string", + }, + "relations": { + "description": "Optional relationships from this new document to other workspace items, at most 20.", + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "description": "\`derived_from\` means this item was created or materially changed from the linked item. \`references\` means this item cites or points to the linked item.", + "enum": [ + "derived_from", + "references", + ], + "type": "string", + }, + "note": { + "description": "Optional short source detail, like pages 12-14 or section on photosynthesis.", + "maxLength": 240, + "type": "string", + }, + "path": { + "description": "Absolute path of the related ThinkEx workspace item.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "kind", + "path", + ], + "type": "object", + }, + "maxItems": 20, + "type": "array", + }, + "type": { + "const": "document", + "type": "string", + }, + }, + "required": [ + "type", + "path", + ], + "type": "object", + }, + ], + }, + "maxItems": 20, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "items", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_delete_items input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "paths": { + "description": "Absolute paths of one or more actual ThinkEx workspace items to delete, at most 20.", + "items": { + "minLength": 1, + "type": "string", + }, + "maxItems": 20, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "paths", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_edit_item input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "edits": { + "description": "Ordered structural HTML edits, at most 40. For targeted operations, copy data-ref into the "ref" field; there is no "target" field. These refs are local to this document and are not workspace citation refs.", + "items": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "html": { + "description": "Schema-constrained HTML fragment. Model-supplied data-ref attributes are ignored.", + "maxLength": 512000, + "type": "string", + }, + "op": { + "enum": [ + "insert_after", + "insert_before", + "replace", + ], + "type": "string", + }, + "ref": { + "description": "Exact data-ref from a recent HTML read. Put it in the "ref" field, never "target".", + "maxLength": 64, + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "html", + "op", + "ref", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "op": { + "const": "delete", + "type": "string", + }, + "ref": { + "description": "Exact data-ref from a recent HTML read. Put it in the "ref" field, never "target".", + "maxLength": 64, + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "op", + "ref", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "html": { + "description": "Schema-constrained HTML fragment. Model-supplied data-ref attributes are ignored.", + "maxLength": 512000, + "type": "string", + }, + "op": { + "const": "replace_all", + "type": "string", + }, + }, + "required": [ + "html", + "op", + ], + "type": "object", + }, + ], + }, + "maxItems": 40, + "minItems": 1, + "type": "array", + }, + "path": { + "description": "Absolute path of one actual ThinkEx workspace item to edit.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "path", + "edits", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_link_items input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "path": { + "description": "Absolute path of the workspace item to link from.", + "minLength": 1, + "type": "string", + }, + "relations": { + "description": "Relationships from this item to other workspace items, at most 20.", + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "description": "\`derived_from\` means this item was created or materially changed from the linked item. \`references\` means this item cites or points to the linked item.", + "enum": [ + "derived_from", + "references", + ], + "type": "string", + }, + "note": { + "description": "Optional short source detail, like pages 12-14 or section on photosynthesis.", + "maxLength": 240, + "type": "string", + }, + "path": { + "description": "Absolute path of the related ThinkEx workspace item.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "kind", + "path", + ], + "type": "object", + }, + "maxItems": 20, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "path", + "relations", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_list_items input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "offset": { + "description": "Zero-based item offset. Use nextOffset from the previous result to continue.", + "maximum": 9007199254740991, + "minimum": 0, + "type": "integer", + }, + "path": { + "description": "Absolute path in the actual ThinkEx workspace. Defaults to /.", + "minLength": 1, + "type": "string", + }, + "recursive": { + "description": "Include nested descendants. Defaults to false for immediate children only.", + "type": "boolean", + }, + }, + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_move_items input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "destinationPath": { + "description": "Absolute path of the destination folder. Use / for the workspace root.", + "minLength": 1, + "type": "string", + }, + "paths": { + "description": "Absolute paths of one or more actual ThinkEx workspace items to move, at most 20.", + "items": { + "minLength": 1, + "type": "string", + }, + "maxItems": 20, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "destinationPath", + "paths", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_read_items input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "requests": { + "description": "Ordered workspace content reads, at most 20.", + "items": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "mode": { + "const": "start", + "type": "string", + }, + "path": { + "description": "Absolute path of the workspace item to read.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "path", + "mode", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "mode": { + "const": "pages", + "type": "string", + }, + "path": { + "description": "Absolute path of the workspace item to read.", + "minLength": 1, + "type": "string", + }, + "range": { + "description": "Up to 20 physical pages from an extracted file, like 1, 3, 5-7, or 1,4-6. Defaults to page 1.", + "minLength": 1, + "pattern": "^\\d+(?:\\s*-\\s*\\d+)?(?:\\s*,\\s*\\d+(?:\\s*-\\s*\\d+)?)*$", + "type": "string", + }, + }, + "required": [ + "path", + "mode", + "range", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "cursor": { + "description": "Opaque cursor returned by a previous read.", + "maxLength": 4096, + "minLength": 1, + "type": "string", + }, + "mode": { + "const": "continue", + "type": "string", + }, + "path": { + "description": "Absolute path of the workspace item to read.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "path", + "cursor", + "mode", + ], + "type": "object", + }, + ], + }, + "maxItems": 20, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "requests", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_rename_item input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "name": { + "description": "New user-visible item name.", + "maxLength": 160, + "minLength": 1, + "type": "string", + }, + "path": { + "description": "Absolute path of one actual ThinkEx workspace item to rename.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "name", + "path", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_search input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "path": { + "description": "Optional absolute workspace path. A folder searches recursively; an item searches only that item. Defaults to /.", + "minLength": 1, + "type": "string", + }, + "query": { + "description": "Text or natural-language question to search for in the workspace.", + "maxLength": 1000, + "minLength": 2, + "type": "string", + }, + "types": { + "description": "Optional content types to include. Defaults to documents and files.", + "items": { + "enum": [ + "document", + "file", + ], + "type": "string", + }, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "query", + ], + "type": "object", +} +`; diff --git a/src/features/workspaces/operations/create-items.ts b/src/features/workspaces/operations/create-items.ts index 9c87ea7d1..671b3204d 100644 --- a/src/features/workspaces/operations/create-items.ts +++ b/src/features/workspaces/operations/create-items.ts @@ -2,8 +2,8 @@ import { getAuthorizedWorkspaceKernel } from "#/features/workspaces/operations/w import { resolveWorkspaceRelations, type WorkspaceRelationInput, - workspaceRelationFailureCodes, } from "#/features/workspaces/operations/relations"; +import { createWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/workspace-operation-failure-codes"; import type { WorkspaceAccessContext } from "#/features/workspaces/operations/workspace-access-context"; import type { WorkspaceKernelPathResolution } from "#/features/workspaces/kernel/workspace-kernel-types"; import { parseDocumentAiHtml } from "#/features/workspaces/documents/document-ai-html"; @@ -32,17 +32,6 @@ export interface CreateWorkspaceItemsOperationInput { items: CreateWorkspaceItemOperationInput[]; } -export const createWorkspaceItemsFailureCodes = [ - "cannot_create_root", - "invalid_initial_content", - "path_already_exists", - "path_not_absolute", - "path_not_canonical", - "path_not_folder", - "path_not_found", - ...workspaceRelationFailureCodes, -] as const; - type CreateWorkspaceItemsFailureCode = (typeof createWorkspaceItemsFailureCodes)[number]; export interface CreateWorkspaceItemsFailure { diff --git a/src/features/workspaces/operations/delete-items.ts b/src/features/workspaces/operations/delete-items.ts index 4064db94a..7ac7e7fe0 100644 --- a/src/features/workspaces/operations/delete-items.ts +++ b/src/features/workspaces/operations/delete-items.ts @@ -9,11 +9,7 @@ export interface DeleteWorkspaceItemsOperationInput { paths: string[]; } -export const deleteWorkspaceItemsFailureCodes = [ - "cannot_delete_root", - "path_not_absolute", - "path_not_found", -] as const; +import { deleteWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/workspace-operation-failure-codes"; export interface DeleteWorkspaceItemsFailure { code: (typeof deleteWorkspaceItemsFailureCodes)[number]; diff --git a/src/features/workspaces/operations/edit-item.ts b/src/features/workspaces/operations/edit-item.ts index b3029eca4..7613e7942 100644 --- a/src/features/workspaces/operations/edit-item.ts +++ b/src/features/workspaces/operations/edit-item.ts @@ -4,23 +4,11 @@ import { resolveWorkspaceExistingItemPath, } from "#/features/workspaces/operations/workspace-operation-context"; import type { WorkspaceAccessContext } from "#/features/workspaces/operations/workspace-access-context"; -import { - type DocumentAiEdit, - documentAiEditFailureCodes, -} from "#/features/workspaces/documents/document-ai-edits"; +import { type DocumentAiEdit } from "#/features/workspaces/documents/document-ai-edits"; +import { editWorkspaceItemFailureCodes } from "#/features/workspaces/operations/workspace-operation-failure-codes"; import type { DocumentEditLineChanges } from "#/features/workspaces/documents/document-edit-receipt"; import { resolveDocumentCitations } from "#/features/workspaces/operations/document-citations"; -export const editWorkspaceItemFailureCodes = [ - "cannot_edit_root", - "path_not_absolute", - "path_not_found", - "unsupported_item_type", - ...documentAiEditFailureCodes, - "content_changed", - "operation_id_conflict", -] as const; - type EditWorkspaceItemFailureCode = (typeof editWorkspaceItemFailureCodes)[number]; export interface EditWorkspaceItemOperationInput { diff --git a/src/features/workspaces/operations/link-items.ts b/src/features/workspaces/operations/link-items.ts index 6e8e42e04..087842229 100644 --- a/src/features/workspaces/operations/link-items.ts +++ b/src/features/workspaces/operations/link-items.ts @@ -2,8 +2,8 @@ import type { WorkspaceItemSummary } from "#/features/workspaces/contracts"; import { resolveWorkspaceRelations, type WorkspaceRelationInput, - workspaceRelationFailureCodes, } from "#/features/workspaces/operations/relations"; +import { linkWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/workspace-operation-failure-codes"; import type { WorkspaceAccessContext } from "#/features/workspaces/operations/workspace-access-context"; import { getAuthorizedWorkspaceKernel, @@ -15,13 +15,6 @@ export interface LinkWorkspaceItemsOperationInput { relations: WorkspaceRelationInput[]; } -export const linkWorkspaceItemsFailureCodes = [ - "cannot_link_root", - "path_not_absolute", - "path_not_found", - ...workspaceRelationFailureCodes, -] as const; - type LinkWorkspaceItemsFailureCode = (typeof linkWorkspaceItemsFailureCodes)[number]; interface LinkWorkspaceItemsFailure { diff --git a/src/features/workspaces/operations/move-items.ts b/src/features/workspaces/operations/move-items.ts index a5fa7aff2..840c59ef0 100644 --- a/src/features/workspaces/operations/move-items.ts +++ b/src/features/workspaces/operations/move-items.ts @@ -15,17 +15,7 @@ export interface MoveWorkspaceItemsOperationInput { paths: string[]; } -export const moveWorkspaceItemsFailureCodes = [ - "already_in_destination", - "cannot_move_into_descendant", - "cannot_move_root", - "destination_path_not_absolute", - "destination_path_not_folder", - "destination_path_not_found", - "path_already_exists", - "path_not_absolute", - "path_not_found", -] as const; +import { moveWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/workspace-operation-failure-codes"; interface MoveWorkspaceDestinationFailure { code: diff --git a/src/features/workspaces/operations/rename-item.ts b/src/features/workspaces/operations/rename-item.ts index ddffd801f..79cd9f784 100644 --- a/src/features/workspaces/operations/rename-item.ts +++ b/src/features/workspaces/operations/rename-item.ts @@ -14,12 +14,7 @@ export interface RenameWorkspaceItemOperationInput { path: string; } -export const renameWorkspaceItemFailureCodes = [ - "cannot_rename_root", - "path_already_exists", - "path_not_absolute", - "path_not_found", -] as const; +import { renameWorkspaceItemFailureCodes } from "#/features/workspaces/operations/workspace-operation-failure-codes"; export interface RenameWorkspaceItemFailure { code: (typeof renameWorkspaceItemFailureCodes)[number]; diff --git a/src/features/workspaces/operations/workspace-operation-failure-codes.ts b/src/features/workspaces/operations/workspace-operation-failure-codes.ts new file mode 100644 index 000000000..6294c9d7b --- /dev/null +++ b/src/features/workspaces/operations/workspace-operation-failure-codes.ts @@ -0,0 +1,60 @@ +import { documentAiEditFailureCodes } from "#/features/workspaces/documents/document-ai-edits"; +import { workspaceRelationFailureCodes } from "#/features/workspaces/operations/relations"; + +// The failure-code vocabulary for each workspace operation. This is a pure leaf +// (no operation impls, no kernel) so the model-facing schema + contract layer can +// depend on it without dragging the worker runtime — which also lets these run in +// plain Node evals. Impls import their codes from here rather than owning them. + +export const createWorkspaceItemsFailureCodes = [ + "cannot_create_root", + "invalid_initial_content", + "path_already_exists", + "path_not_absolute", + "path_not_canonical", + "path_not_folder", + "path_not_found", + ...workspaceRelationFailureCodes, +] as const; + +export const deleteWorkspaceItemsFailureCodes = [ + "cannot_delete_root", + "path_not_absolute", + "path_not_found", +] as const; + +export const editWorkspaceItemFailureCodes = [ + "cannot_edit_root", + "path_not_absolute", + "path_not_found", + "unsupported_item_type", + ...documentAiEditFailureCodes, + "content_changed", + "operation_id_conflict", +] as const; + +export const linkWorkspaceItemsFailureCodes = [ + "cannot_link_root", + "path_not_absolute", + "path_not_found", + ...workspaceRelationFailureCodes, +] as const; + +export const moveWorkspaceItemsFailureCodes = [ + "already_in_destination", + "cannot_move_into_descendant", + "cannot_move_root", + "destination_path_not_absolute", + "destination_path_not_folder", + "destination_path_not_found", + "path_already_exists", + "path_not_absolute", + "path_not_found", +] as const; + +export const renameWorkspaceItemFailureCodes = [ + "cannot_rename_root", + "path_already_exists", + "path_not_absolute", + "path_not_found", +] as const; diff --git a/src/features/workspaces/operations/workspace-tool-schemas.ts b/src/features/workspaces/operations/workspace-tool-schemas.ts index 1cc79ffa5..26d3a91fb 100644 --- a/src/features/workspaces/operations/workspace-tool-schemas.ts +++ b/src/features/workspaces/operations/workspace-tool-schemas.ts @@ -4,12 +4,14 @@ import { workspaceReadItemsInputSchema, workspaceReadItemsOutputSchema, } from "#/features/workspaces/content/workspace-content-contract"; -import { createWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/create-items"; -import { deleteWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/delete-items"; -import { editWorkspaceItemFailureCodes } from "#/features/workspaces/operations/edit-item"; -import { linkWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/link-items"; -import { moveWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/move-items"; -import { renameWorkspaceItemFailureCodes } from "#/features/workspaces/operations/rename-item"; +import { + createWorkspaceItemsFailureCodes, + deleteWorkspaceItemsFailureCodes, + editWorkspaceItemFailureCodes, + linkWorkspaceItemsFailureCodes, + moveWorkspaceItemsFailureCodes, + renameWorkspaceItemFailureCodes, +} from "#/features/workspaces/operations/workspace-operation-failure-codes"; import { workspaceItemTypeSchema, workspaceRelationKindSchema, @@ -33,7 +35,7 @@ export { }; export const workspaceDocumentHtmlInstruction = - 'Use semantic HTML with paragraphs, h1-h4, blockquotes, lists, code blocks, horizontal rules, tables, links, and standard text marks. For math, use or
. For checkboxes, use

  • Item

. Documents cannot hold images: never use or
, and describe the visual in words instead. Cite workspace sources in documents exactly as in a chat reply, with placed after the claim it supports.'; + 'Use semantic HTML with paragraphs, h1-h4, blockquotes, lists, code blocks, horizontal rules, tables, links, and standard text marks. For math, use or
— this is the only math that renders, so never write $...$, $$...$$, or \\(...\\) in the HTML, and never put dollar signs around the data-latex value. Put every subscript and superscript (exponents like 10^8, indices like x_1) inside math, not or tags, which are unsupported and reject the whole write. Chemistry renders with \\ce{...} (e.g. \\ce{CH4 + 2 O2 -> CO2 + 2 H2O}) and quantities with units render with \\pu{...} (e.g. \\pu{9.81 m/s^2}), both inside data-latex. Write literal money as plain text ($30, never \\$30) — this is HTML, not Markdown, so a backslash before a dollar sign shows on screen. For checkboxes, use

  • Item

. Documents cannot hold images: never use or
, and describe the visual in words instead. Cite workspace sources in documents exactly as in a chat reply, with placed after the claim it supports.'; const workspacePathSchema = z.string().min(1); const workspaceIndexSchema = z.number().int().nonnegative(); @@ -178,9 +180,9 @@ export const workspaceCreateItemsInputSchema = z.object({ "Optional relationships from this new document to other workspace items, at most 20.", ), initialContent: documentAiHtmlSchema - .describe( - `Optional initial HTML content for the document. ${workspaceDocumentHtmlInstruction}`, - ) + // Doc HTML rules live in the create tool description (see + // workspace-tool-definitions), so they are not repeated here. + .describe("Optional initial HTML content for the document.") .optional(), }), ]), diff --git a/src/features/workspaces/operations/workspace-tool-surface.test.ts b/src/features/workspaces/operations/workspace-tool-surface.test.ts new file mode 100644 index 000000000..b24605785 --- /dev/null +++ b/src/features/workspaces/operations/workspace-tool-surface.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { getAIThreadSoulPrompt } from "#/features/workspaces/ai/ai-thread-soul-prompt"; +import { + workspaceCreateItemsInputSchema, + workspaceDeleteItemsInputSchema, + workspaceEditItemInputSchema, + workspaceLinkItemsInputSchema, + workspaceListItemsInputSchema, + workspaceMoveItemsInputSchema, + workspaceReadItemsInputSchema, + workspaceRenameItemInputSchema, + workspaceSearchInputSchema, +} from "#/features/workspaces/operations/workspace-tool-schemas"; + +// Free, deterministic regression net for the model-facing surface: the assembled +// system prompt and each tool's input JSON schema (field `.describe()` text +// included). A change to a schema shape, a field description, or the soul prompt +// shows up here as a reviewable diff — zero model calls, runs in normal CI. +// +// Update intentionally with `pnpm test -u` when the change is deliberate; an +// unexpected diff is a regression to look at. Whether the model actually *uses* +// the surface correctly is the live evals' job: `pnpm eval`. + +const TOOL_INPUT_SCHEMAS = { + workspace_list_items: workspaceListItemsInputSchema, + workspace_read_items: workspaceReadItemsInputSchema, + workspace_search: workspaceSearchInputSchema, + workspace_create_items: workspaceCreateItemsInputSchema, + workspace_edit_item: workspaceEditItemInputSchema, + workspace_delete_items: workspaceDeleteItemsInputSchema, + workspace_move_items: workspaceMoveItemsInputSchema, + workspace_rename_item: workspaceRenameItemInputSchema, + workspace_link_items: workspaceLinkItemsInputSchema, +} as const; + +describe("workspace tool surface", () => { + it("system prompt is stable", () => { + expect(getAIThreadSoulPrompt()).toMatchSnapshot(); + }); + + for (const [name, schema] of Object.entries(TOOL_INPUT_SCHEMAS)) { + it(`${name} input schema is stable`, () => { + expect(z.toJSONSchema(schema)).toMatchSnapshot(); + }); + } +}); diff --git a/tsconfig.json b/tsconfig.json index 36eb548e9..a62ad5b0a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,9 +3,11 @@ "src/**/*.ts", "src/**/*.tsx", "test/**/*.ts", + "eval/**/*.ts", "drizzle.config.ts", "vite.config.ts", - "vitest.config.ts" + "vitest.config.ts", + "vitest.evals.config.ts" ], "compilerOptions": { "target": "ES2022", diff --git a/vitest.evals.config.ts b/vitest.evals.config.ts new file mode 100644 index 000000000..97ee23683 --- /dev/null +++ b/vitest.evals.config.ts @@ -0,0 +1,57 @@ +import { fileURLToPath } from "node:url"; + +import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; +import agents from "agents/vite"; +import { defineConfig } from "vite-plus/test/config"; + +// Evals exercise the real workspace tool surface (schemas + descriptions) and the +// real gateway model wiring. That code is worker-runtime code — it imports the +// kernel, Durable Objects, and `cloudflare:workers` — so, exactly like the app's +// own `*.worker.test.ts`, evals run in the Cloudflare workers pool rather than +// Node. This gives us the real modules with no stubbing. +// +// They live in their own config so they never run in the normal `pnpm test` +// pipeline: they hit real models (real latency + spend). Run them on demand with +// `pnpm eval`, which injects AI_GATEWAY_API_KEY (via Infisical) into the pool. +export default defineConfig({ + test: { + projects: [ + { + resolve: { + tsconfigPaths: true, + alias: { + // The worker entry pulls in the TanStack Start server handler, whose + // build-time virtual specifiers don't resolve under Vitest. The app's + // own workers project stubs it the same way. + "@tanstack/react-start/server": fileURLToPath( + new URL("./test/stubs/tanstack-start-server.ts", import.meta.url), + ), + }, + }, + plugins: [ + // Agent classes use TC39 decorators that Oxc can't transform yet; this + // plugin lowers them (same as the app build and workers test project). + agents(), + cloudflareTest({ + main: "./test/worker-entry.ts", + remoteBindings: false, + wrangler: { configPath: "./wrangler.jsonc" }, + // AI_GATEWAY_API_KEY is a secret, so it isn't in wrangler vars. Inject + // it from the process env (Infisical supplies it under `pnpm eval`) so + // the real gateway wiring can authenticate. + miniflare: { + bindings: { AI_GATEWAY_API_KEY: process.env.AI_GATEWAY_API_KEY ?? "" }, + }, + }), + ], + test: { + name: "evals", + include: ["eval/**/*.eval.ts"], + // Real model turns are slow; give them room. + testTimeout: 120_000, + hookTimeout: 120_000, + }, + }, + ], + }, +});