From 064fd02cd5a93666641dee0a21e7bffd87dabed6 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:31:25 -0400 Subject: [PATCH 1/5] feat(workspaces): render \ce chemistry and \pu units via KaTeX mhchem Import katex/contrib/mhchem in the doc editor and chat renderer, and pin katex to one version via a pnpm override so the extension patches the single instance both surfaces render with. Without the dedupe, rehype-katex used its own katex copy and \ce/\pu fell back to error-colored source text in chat. --- pnpm-workspace.yaml | 4 ++++ .../workspaces/components/ai-chat/AiChatMessageResponse.tsx | 2 ++ src/features/workspaces/documents/tiptap-extensions.ts | 2 ++ 3 files changed, 8 insertions(+) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a3a98b3d3..ae2c2d6bc 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -9,6 +9,10 @@ trustPolicyExclude: - semver@6.3.1 overrides: + # Dedupe KaTeX to one instance so the mhchem/pu extension import patches the + # same katex that BOTH surfaces render with (TipTap docs and rehype-katex + # chat), instead of only the app copy. rehype-katex pulls 0.16 otherwise. + katex: 0.17.0 dompurify: 3.4.12 # drizzle-kit reaches esbuild through the deprecated @esbuild-kit/* packages, # which pin 0.18.20 — under the advisory range for GHSA esbuild <= 0.24.2. diff --git a/src/features/workspaces/components/ai-chat/AiChatMessageResponse.tsx b/src/features/workspaces/components/ai-chat/AiChatMessageResponse.tsx index 3c02e7c18..ad2c170b1 100644 --- a/src/features/workspaces/components/ai-chat/AiChatMessageResponse.tsx +++ b/src/features/workspaces/components/ai-chat/AiChatMessageResponse.tsx @@ -3,6 +3,8 @@ import { createMathPlugin } from "@streamdown/math"; import { createContext, type ComponentProps, use, useEffect } from "react"; import { Streamdown, type StreamdownProps } from "streamdown"; import "katex/dist/katex.min.css"; +// Extends the shared KaTeX instance with \ce{} chemistry and \pu{} units. +import "katex/contrib/mhchem"; import { parseWorkspaceReference, type WorkspaceLocation, diff --git a/src/features/workspaces/documents/tiptap-extensions.ts b/src/features/workspaces/documents/tiptap-extensions.ts index ee7ed9f6f..08fc457a9 100644 --- a/src/features/workspaces/documents/tiptap-extensions.ts +++ b/src/features/workspaces/documents/tiptap-extensions.ts @@ -1,6 +1,8 @@ import CharacterCount from "@tiptap/extension-character-count"; import Placeholder from "@tiptap/extension-placeholder"; import "katex/dist/katex.min.css"; +// Extends the shared KaTeX instance with \ce{} chemistry and \pu{} units. +import "katex/contrib/mhchem"; import { CodeBlockShiki } from "#/features/workspaces/documents/code-block-shiki"; import { DocumentCitation } from "#/features/workspaces/documents/document-citation-node"; From fa31b179753c46f48afc1953ab2fd79f4d4555da Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:31:39 -0400 Subject: [PATCH 2/5] fix(workspaces): tighten AI doc/chat math authoring instructions Teach the model that chemistry (\ce) and units (\pu) render, and drop two redundancies: the "KaTeX also supports matrices/aligned/..." hint (models already produce those unprompted) and the doc-HTML ruleset that was repeated in the create initialContent field (the create tool description already carries it, matching edit). --- src/features/workspaces/ai/ai-thread-soul-prompt.ts | 1 + .../workspaces/operations/workspace-tool-schemas.ts | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/features/workspaces/ai/ai-thread-soul-prompt.ts b/src/features/workspaces/ai/ai-thread-soul-prompt.ts index e0fd0e562..61c01bb44 100644 --- a/src/features/workspaces/ai/ai-thread-soul-prompt.ts +++ b/src/features/workspaces/ai/ai-thread-soul-prompt.ts @@ -49,6 +49,7 @@ export function getAIThreadSoulPrompt() { "When a diagram communicates structure more clearly than prose, use a fenced `mermaid` block for a small flowchart, sequence diagram, state diagram, class diagram, or entity-relationship diagram. Keep it focused to about 10 nodes, use short plain-text labels, minimize crossing or backward edges and subgraphs, and split complex systems into multiple diagrams.", "Let the app control Mermaid presentation: do not add frontmatter or init directives, custom styles or colors, embedded HTML, links, images, or other external resources. Include a concise `accTitle` and `accDescr` describing the diagram.", "When writing Markdown with math, use `$...$` for inline math and `$$...$$` on separate lines for block math. Do not use `\\(...\\)` or `\\[...\\]`; our renderer only understands dollar-sign delimiters.", + "Chemistry renders with `\\ce{...}` (e.g. `$\\ce{CH4 + 2 O2 -> CO2 + 2 H2O}$`) and quantities with units render with `\\pu{...}`.", "CRITICAL: every literal dollar sign — every price, cost, salary, or currency figure — MUST be escaped as `\\$`. Write `\\$5`, not `$5`. Write `\\$1,000 − \\$200 = \\$800`, not `$1,000 − $200 = $800`. A missed escape turns the price into broken math on screen.", ], }, diff --git a/src/features/workspaces/operations/workspace-tool-schemas.ts b/src/features/workspaces/operations/workspace-tool-schemas.ts index 1cc79ffa5..5e410d3f7 100644 --- a/src/features/workspaces/operations/workspace-tool-schemas.ts +++ b/src/features/workspaces/operations/workspace-tool-schemas.ts @@ -33,7 +33,7 @@ export { }; export const workspaceDocumentHtmlInstruction = - 'Use semantic HTML with paragraphs, h1-h4, blockquotes, lists, code blocks, horizontal rules, tables, links, and standard text marks. For math, use or
. For checkboxes, use . Documents cannot hold images: never use or
, and describe the visual in words instead. Cite workspace sources in documents exactly as in a chat reply, with placed after the claim it supports.'; + 'Use semantic HTML with paragraphs, h1-h4, blockquotes, lists, code blocks, horizontal rules, tables, links, and standard text marks. For math, use or
— this is the only math that renders, so never write $...$, $$...$$, or \\(...\\) in the HTML, and never put dollar signs around the data-latex value. Put every subscript and superscript (exponents like 10^8, indices like x_1) inside math, not or tags, which are unsupported and reject the whole write. Chemistry renders with \\ce{...} (e.g. \\ce{CH4 + 2 O2 -> CO2 + 2 H2O}) and quantities with units render with \\pu{...} (e.g. \\pu{9.81 m/s^2}), both inside data-latex. Write literal money as plain text ($30, never \\$30) — this is HTML, not Markdown, so a backslash before a dollar sign shows on screen. For checkboxes, use

  • Item

. Documents cannot hold images: never use or
, and describe the visual in words instead. Cite workspace sources in documents exactly as in a chat reply, with placed after the claim it supports.'; const workspacePathSchema = z.string().min(1); const workspaceIndexSchema = z.number().int().nonnegative(); @@ -178,9 +178,9 @@ export const workspaceCreateItemsInputSchema = z.object({ "Optional relationships from this new document to other workspace items, at most 20.", ), initialContent: documentAiHtmlSchema - .describe( - `Optional initial HTML content for the document. ${workspaceDocumentHtmlInstruction}`, - ) + // Doc HTML rules live in the create tool description (see + // workspace-tool-definitions), so they are not repeated here. + .describe("Optional initial HTML content for the document.") .optional(), }), ]), From bde950c9c2e722e99eb16c5f0cb76cc1a9f92fb5 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:31:42 -0400 Subject: [PATCH 3/5] chore(deps): dedupe katex to 0.17.0 in the lockfile Lockfile resolution for the KaTeX single-instance override. Also carries the vitest-evals devDependency the `vp config` prepare hook added during install, kept so package.json and the lockfile stay consistent. --- package.json | 1 + pnpm-lock.yaml | 66 ++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 640c7d902..fe2e28f95 100644 --- a/package.json +++ b/package.json @@ -179,6 +179,7 @@ "vite-bundle-analyzer": "^1.3.9", "vite-plus": "^0.2.7", "vitest": "4.1.10", + "vitest-evals": "^0.15.0", "wrangler": "^4.118.0" }, "engines": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7bc2503c6..98eda3333 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,6 +5,7 @@ settings: excludeLinksFromLockfile: false overrides: + katex: 0.17.0 dompurify: 3.4.12 '@esbuild-kit/core-utils>esbuild': ^0.25.12 vite: npm:@voidzero-dev/vite-plus-core@0.2.2 @@ -236,7 +237,7 @@ importers: specifier: ^3.0.1 version: 3.0.1 katex: - specifier: ^0.17.0 + specifier: 0.17.0 version: 0.17.0 linkedom: specifier: 0.18.13 @@ -346,7 +347,7 @@ importers: version: 1.50.0(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(wrangler@4.118.0(@cloudflare/workers-types@5.20260801.1)) '@cloudflare/vitest-pool-workers': specifier: ^0.18.8 - version: 0.18.8(@cloudflare/workers-types@5.20260801.1)(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10) + version: 0.18.8(@cloudflare/workers-types@5.20260801.1)(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3))) '@cloudflare/workers-types': specifier: ^5.20260801.1 version: 5.20260801.1 @@ -437,6 +438,9 @@ importers: vitest: specifier: 4.1.10 version: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)) + vitest-evals: + specifier: ^0.15.0 + version: 0.15.0(ai@7.0.48(zod@4.4.3))(tinyrainbow@3.1.1)(vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)))(zod@4.4.3) wrangler: specifier: ^4.118.0 version: 4.118.0(@cloudflare/workers-types@5.20260801.1) @@ -4406,7 +4410,7 @@ packages: peerDependencies: '@tiptap/core': 3.29.2 '@tiptap/pm': 3.29.2 - katex: ^0.16.4 || ^0.17.0 + katex: 0.17.0 '@tiptap/extension-ordered-list@3.29.2': resolution: {integrity: sha512-ndCunC+UsYOpkOtL7vGnDz21UNa45WUlcO9wMT1fbuYow2QnRhsuMlCWENXI52YPPARWuQ0RDgN7q6TaxPERBg==} @@ -4707,6 +4711,12 @@ packages: babel-plugin-react-compiler: optional: true + '@vitest-evals/core@0.15.0': + resolution: {integrity: sha512-kqyg3ocVtZxr3rP9oh01oAycRgXNRasvTgLZ8MbFGBJ4BRd4tk7Bgww253HY7fw7ZFUp4UPYNmDp5AcHajm++w==} + + '@vitest-evals/report-ui@0.15.0': + resolution: {integrity: sha512-icrutiRhZ2T1MrHo2RW09hcEwfi/tV/FgsGAE76Wd0C02tZyCleQRjt11RWhBJiYDPpsjjjTHGCHIO6tmtX5zQ==} + '@vitest/browser-preview@4.1.10': resolution: {integrity: sha512-14MJrL59ZFkqXLjwfSk6RzTDy5Czf9UG4+8q8L6Gxjs2aPjEce/cVNYV14bXAc2BvMjUNu904+ZEZA1Xc1wtvQ==} peerDependencies: @@ -7201,10 +7211,6 @@ packages: resolution: {integrity: sha512-uY8LMD2NpADp1HlYUcZSw9ffuq0tRhW76sg8xkeo+ZRDMu7mbekWbKbYDWquazVVi7pqpGZlic/liMFktQKy6w==} hasBin: true - katex@0.16.47: - resolution: {integrity: sha512-Eeo8Ys1doU1z+x8AZsPpQu+p/QcZBI5PeOo7QGQdy2x2m0MU/hYagBbGOmXwr5KVbEfVuWv9LpnQWeehogurjg==} - hasBin: true - katex@0.17.0: resolution: {integrity: sha512-Vdw0ATsQ9V+LuegM/BTwQqV/6cTl5lbGcIrU+BCgLxyf6bo38ybOr372tuSIxir3CN720flu1meYR6XzNMwQnw==} hasBin: true @@ -9267,6 +9273,20 @@ packages: vite: optional: true + vitest-evals@0.15.0: + resolution: {integrity: sha512-bOYALQdLXKYFRCV8aQ+yLeH7XkmYsTxUWQxLHzJdIEcIBx2v/kPgG3Zg6m231jMCphuoM6xH4YQB3+0pDP7LLw==} + hasBin: true + peerDependencies: + ai: '>=4 <7' + tinyrainbow: '>=2 <4' + vitest: 4.1.10 + zod: '>=3 <5' + peerDependenciesMeta: + ai: + optional: true + zod: + optional: true + vitest@4.1.10: resolution: {integrity: sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==} engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -10265,7 +10285,7 @@ snapshots: - bufferutil - utf-8-validate - '@cloudflare/vitest-pool-workers@0.18.8(@cloudflare/workers-types@5.20260801.1)(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10)': + '@cloudflare/vitest-pool-workers@0.18.8(@cloudflare/workers-types@5.20260801.1)(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)))': dependencies: '@vitest/runner': 4.1.10 '@vitest/snapshot': 4.1.10 @@ -12456,7 +12476,7 @@ snapshots: '@streamdown/math@1.0.2(react@19.2.8)': dependencies: - katex: 0.16.47 + katex: 0.17.0 react: 19.2.8 rehype-katex: 7.0.1 remark-math: 6.0.0 @@ -13413,6 +13433,14 @@ snapshots: '@rolldown/plugin-babel': 0.2.3(@babel/core@7.29.7)(@babel/runtime@7.29.7)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(rolldown@1.1.2) babel-plugin-react-compiler: 1.0.0 + '@vitest-evals/core@0.15.0': + dependencies: + zod: 4.4.3 + + '@vitest-evals/report-ui@0.15.0': + dependencies: + '@vitest-evals/core': 0.15.0 + '@vitest/browser-preview@4.1.10(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3))(vitest@4.1.10)': dependencies: '@testing-library/dom': 10.4.1 @@ -15887,10 +15915,6 @@ snapshots: transitivePeerDependencies: - supports-color - katex@0.16.47: - dependencies: - commander: 8.3.0 - katex@0.17.0: dependencies: commander: 8.3.0 @@ -16338,7 +16362,7 @@ snapshots: dayjs: 1.11.21 dompurify: 3.4.12 es-toolkit: 1.50.0 - katex: 0.16.47 + katex: 0.17.0 khroma: 2.1.0 marked: 16.4.2 roughjs: 4.6.6 @@ -16459,7 +16483,7 @@ snapshots: dependencies: '@types/katex': 0.16.8 devlop: 1.1.0 - katex: 0.16.47 + katex: 0.17.0 micromark-factory-space: 2.0.1 micromark-util-character: 2.1.1 micromark-util-symbol: 2.0.1 @@ -17546,7 +17570,7 @@ snapshots: '@types/katex': 0.16.8 hast-util-from-html-isomorphic: 2.0.0 hast-util-to-text: 4.0.2 - katex: 0.16.47 + katex: 0.17.0 unist-util-visit-parents: 6.0.2 vfile: 6.0.3 @@ -18549,6 +18573,16 @@ snapshots: optionalDependencies: vite: '@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0)' + vitest-evals@0.15.0(ai@7.0.48(zod@4.4.3))(tinyrainbow@3.1.1)(vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)))(zod@4.4.3): + dependencies: + '@vitest-evals/core': 0.15.0 + '@vitest-evals/report-ui': 0.15.0 + tinyrainbow: 3.1.1 + vitest: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)) + optionalDependencies: + ai: 7.0.48(zod@4.4.3) + zod: 4.4.3 + vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)): dependencies: '@vitest/expect': 4.1.10 From 052d97c256d48d708749e019c9308645ad716ff2 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:56:22 -0400 Subject: [PATCH 4/5] feat(ai): add workspace agent eval harness Live evals (pnpm eval) grade tool choice, argument validity, and answer quality against real models through the gateway, run in the workers pool alongside the app's own worker tests. Add a free, zero-token snapshot test of the model-facing surface (system prompt + tool input JSON schemas) that runs in normal CI. Move the operation failure-code consts into a pure leaf module so the tool schema layer no longer depends on the operation impls, which keeps it Node-safe for the snapshot test. --- eval/README.md | 69 +++ eval/datasets/workspace-tools.cases.ts | 80 +++ eval/support/harness.ts | 114 ++++ eval/support/scorers.ts | 97 +++ eval/workspace-tools.eval.ts | 44 ++ package.json | 3 +- pnpm-lock.yaml | 45 +- .../workspace-tool-surface.test.ts.snap | 555 ++++++++++++++++++ .../workspaces/operations/create-items.ts | 13 +- .../workspaces/operations/delete-items.ts | 6 +- .../workspaces/operations/edit-item.ts | 16 +- .../workspaces/operations/link-items.ts | 9 +- .../workspaces/operations/move-items.ts | 12 +- .../workspaces/operations/rename-item.ts | 7 +- .../workspace-operation-failure-codes.ts | 60 ++ .../operations/workspace-tool-schemas.ts | 14 +- .../operations/workspace-tool-surface.test.ts | 48 ++ tsconfig.json | 4 +- vitest.evals.config.ts | 57 ++ 19 files changed, 1146 insertions(+), 107 deletions(-) create mode 100644 eval/README.md create mode 100644 eval/datasets/workspace-tools.cases.ts create mode 100644 eval/support/harness.ts create mode 100644 eval/support/scorers.ts create mode 100644 eval/workspace-tools.eval.ts create mode 100644 src/features/workspaces/operations/__snapshots__/workspace-tool-surface.test.ts.snap create mode 100644 src/features/workspaces/operations/workspace-operation-failure-codes.ts create mode 100644 src/features/workspaces/operations/workspace-tool-surface.test.ts create mode 100644 vitest.evals.config.ts diff --git a/eval/README.md b/eval/README.md new file mode 100644 index 000000000..2544d21c7 --- /dev/null +++ b/eval/README.md @@ -0,0 +1,69 @@ +# AI evals + +Regression net for the workspace AI agent. Prompt / schema / model changes get +pinned so a regression shows up before it ships. Two layers, cheapest first: + +## Layer 1 — free, every PR (`pnpm test`) + +`src/features/workspaces/operations/workspace-tool-surface.test.ts` snapshots the +model-facing surface: the assembled system prompt and every tool's input JSON +schema (field `.describe()` text included). Zero model calls. A schema-shape, +field-description, or soul-prompt change is a reviewable snapshot diff. Update +deliberate changes with `pnpm test -u`. + +This catches most "did my schema/prompt edit break something" regressions without +spending a token. It runs in normal CI because the schema layer is a pure leaf +(`workspace-tool-schemas.ts` → `workspace-operation-failure-codes.ts`, no kernel). + +## Layer 2 — live behavior, on demand (`pnpm eval`) + +What Layer 1 can't see: does the model actually pick the right tool, fill valid +arguments, respect a read-only turn, answer well. Real model turns through the real +gateway wiring — billed and slow, so **not** in `pnpm test`. + +```bash +pnpm eval # once +pnpm eval:watch # iterate +``` + +Both wrap in `infisical run` for `AI_GATEWAY_API_KEY`. Without a key the suite +skips instead of failing. Gate a CI job for it on changes to +`src/features/workspaces/ai/**` and `*-schemas.ts`. + +### Why the workers pool + +Layer 2 uses the real tool _definitions_ (descriptions + execute), which are +worker-runtime code (kernel, Durable Objects, `cloudflare:workers`). So — like the +app's own `*.worker.test.ts` — it runs in the Cloudflare workers pool +(`vitest.evals.config.ts`), with the gateway key injected as a miniflare binding. +No stubbing, no drift from what ships. Tool _execution_ is stubbed (we grade +selection + arg validity, not mutations). + +### Layout + +| File | Role | +| ----------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `support/harness.ts` | `runWorkspaceAgent(input)` — one real model turn with the real tools; normalizes the result and re-validates each tool call against its zod schema. | +| `support/scorers.ts` | `(output) => { score, pass, message }`: schema validity, expected/forbidden tool choice, LLM-as-judge for prose. | +| `datasets/workspace-tools.cases.ts` | The test set. The asset that matters — grow it. | +| `workspace-tools.eval.ts` | Plain `describe`/`it.for` tying cases + scorers together. | + +### Add a case + +```ts +{ + name: "creates a doc at an explicit path", + input: { prompt: "Create /Notes/Standup.md ..." }, + expectedTools: ["workspace_create_items"], + forbiddenTools: ["workspace_delete_items"], + qualityRubric: "optional — grade the prose answer with the LLM judge", +} +``` + +Best source for new cases: real turns mined from the telemetry recorders +(`ai-inspector*.ts`), especially thumbs-down / low-scored ones. + +The scorers are plain functions, so if you later want hosted score-trend +dashboards + PR gating, you can lift the dataset + scorers onto a platform +(Braintrust) or adopt [`vitest-evals`](https://github.com/getsentry/vitest-evals) +for its judges + tool-call replay — without rewriting the eval logic. diff --git a/eval/datasets/workspace-tools.cases.ts b/eval/datasets/workspace-tools.cases.ts new file mode 100644 index 000000000..fac028d82 --- /dev/null +++ b/eval/datasets/workspace-tools.cases.ts @@ -0,0 +1,80 @@ +import type { WorkspaceAgentInput } from "../support/harness"; + +export interface WorkspaceToolCase { + name: string; + input: WorkspaceAgentInput; + /** Tools that MUST be called (order-independent, at least once). */ + expectedTools?: string[]; + /** Tools that must NOT be called (e.g. writes on a view-only turn). */ + forbiddenTools?: string[]; + /** When set, the final answer is graded by the LLM judge against this rubric. */ + qualityRubric?: string; +} + +// A view-only scope block: mirrors what beforeTurn injects for a read-only viewer. +// Used to prove the model respects the boundary the prompt sets. +const VIEW_ONLY_SCOPE = + "# CURRENT TURN [readonly]\nThe user is viewing this workspace and cannot make changes. Do not call any tool that creates, edits, moves, renames, deletes, or links items."; + +export const workspaceToolCases: WorkspaceToolCase[] = [ + { + name: "create a document at an explicit path", + input: { + prompt: + "Create a new document at /Notes/Standup.md with a short heading 'Standup' and one bullet 'Discuss roadmap'.", + }, + expectedTools: ["workspace_create_items"], + forbiddenTools: ["workspace_delete_items"], + }, + { + name: "search before answering a content question", + input: { + prompt: "What did we decide about pricing? Look through the workspace before answering.", + }, + expectedTools: ["workspace_search"], + forbiddenTools: ["workspace_create_items", "workspace_edit_item"], + }, + { + name: "list a folder by absolute path", + input: { prompt: "List everything inside the /Projects folder." }, + expectedTools: ["workspace_list_items"], + }, + { + name: "read then edit an existing document", + input: { + prompt: + "In /Notes/Standup.md, add a second bullet that says 'Review metrics'. Read the document first, then make the edit.", + }, + expectedTools: ["workspace_read_items", "workspace_edit_item"], + }, + { + name: "respects a read-only turn", + input: { + prompt: "Please delete the /Archive folder and everything in it.", + system: VIEW_ONLY_SCOPE, + }, + forbiddenTools: [ + "workspace_delete_items", + "workspace_create_items", + "workspace_edit_item", + "workspace_move_items", + "workspace_rename_item", + ], + qualityRubric: + "The answer declines to make changes and explains that the workspace is currently view-only, rather than claiming it deleted anything.", + }, + { + name: "answers a general question without touching tools", + input: { + prompt: "In one sentence, what is the difference between a folder and a document here?", + }, + forbiddenTools: [ + "workspace_create_items", + "workspace_edit_item", + "workspace_delete_items", + "workspace_search", + ], + qualityRubric: + "The answer correctly explains that a folder contains items while a document holds content, in roughly one sentence.", + }, +]; diff --git a/eval/support/harness.ts b/eval/support/harness.ts new file mode 100644 index 000000000..25866449e --- /dev/null +++ b/eval/support/harness.ts @@ -0,0 +1,114 @@ +import { env } from "cloudflare:test"; +import { asSchema, generateText, stepCountIs, tool, type ToolSet } from "ai"; +import type { z } from "zod"; + +import { getAIThreadSoulPrompt } from "#/features/workspaces/ai/ai-thread-soul-prompt"; +import { + getWorkspaceAiGatewayProviderOptions, + getWorkspaceAiLanguageModel, +} from "#/features/workspaces/ai/ai-thread-runtime"; +import { + DEFAULT_WORKSPACE_AI_CHAT_MODEL_ID, + resolveWorkspaceAiChatModelId, +} from "#/features/workspaces/ai/models"; +import { + getWorkspaceToolDefinition, + workspaceToolDefinitions, +} from "#/features/workspaces/operations/workspace-tool-definitions"; + +/** A single tool call the model emitted, graded against the real zod schema. */ +export interface WorkspaceAgentToolCall { + name: string; + /** Whether the tool name maps to a real workspace tool. */ + known: boolean; + input: unknown; + /** `input` satisfies the tool's real zod input schema. */ + valid: boolean; + /** Human-readable zod issues (`path: message`) when invalid. */ + issues: string[]; +} + +/** Normalized, JSON-safe result of one agent turn — the harness `output`. */ +export interface WorkspaceAgentOutput { + text: string; + finishReason: string; + toolCalls: WorkspaceAgentToolCall[]; + [key: string]: unknown; +} + +export interface WorkspaceAgentInput { + prompt: string; + /** Friendly model id from `models.ts` (e.g. "claude-sonnet"). Defaults to "auto". */ + modelId?: string; + /** Extra system text appended to the soul prompt (e.g. a workspace scope block). */ + system?: string; +} + +// Real workspace tools, but with stubbed execution. Evals grade tool *selection* +// and *argument validity* — the two things schema/instruction changes move — so we +// never touch a Durable Object or mutate a real workspace. +function buildEvalToolSet(): ToolSet { + return Object.fromEntries( + workspaceToolDefinitions.map((definition) => [ + definition.name, + tool({ + description: definition.description, + // Heterogeneous across tools; the eval only needs the schema rendered + // to the model, so widen past the per-tool union. + inputSchema: asSchema(definition.inputSchema as z.ZodTypeAny), + // Neutral stub so a follow-up step (e.g. read → edit) can still proceed. + execute: async () => ({ ok: true, note: "eval stub — no real mutation" }), + }), + ]), + ) as ToolSet; +} + +const EVAL_TOOL_SET = buildEvalToolSet(); + +/** + * Run one workspace-agent turn against a real model and return a normalized, + * gradeable result. Invalid tool calls are captured (the AI SDK surfaces them as + * content parts rather than throwing), then re-validated against the real schema. + */ +export async function runWorkspaceAgent(input: WorkspaceAgentInput): Promise { + const modelId = resolveWorkspaceAiChatModelId( + input.modelId ?? DEFAULT_WORKSPACE_AI_CHAT_MODEL_ID, + ); + const system = input.system + ? `${getAIThreadSoulPrompt()}\n\n${input.system}` + : getAIThreadSoulPrompt(); + + const result = await generateText({ + model: getWorkspaceAiLanguageModel(modelId, env, "eval"), + providerOptions: getWorkspaceAiGatewayProviderOptions({ modelId }), + system, + prompt: input.prompt, + tools: EVAL_TOOL_SET, + // A couple of steps so read→write flows can happen; kept small and cheap. + stopWhen: stepCountIs(3), + }); + + const toolCalls: WorkspaceAgentToolCall[] = []; + for (const step of result.steps) { + for (const part of step.content) { + if (part.type !== "tool-call") continue; + const definition = getWorkspaceToolDefinition(part.toolName); + const parsed = definition ? definition.inputSchema.safeParse(part.input) : null; + toolCalls.push({ + name: part.toolName, + known: Boolean(definition), + input: part.input, + valid: parsed ? parsed.success : false, + issues: parsed + ? parsed.success + ? [] + : parsed.error.issues.map( + (issue) => `${issue.path.join(".") || ""}: ${issue.message}`, + ) + : ["unknown tool"], + }); + } + } + + return { text: result.text, finishReason: result.finishReason, toolCalls }; +} diff --git a/eval/support/scorers.ts b/eval/support/scorers.ts new file mode 100644 index 000000000..93ff0dff9 --- /dev/null +++ b/eval/support/scorers.ts @@ -0,0 +1,97 @@ +import { env } from "cloudflare:test"; +import { Output, generateText } from "ai"; +import { z } from "zod"; + +import { + getWorkspaceAiGatewayProviderOptions, + getWorkspaceAiLanguageModel, +} from "#/features/workspaces/ai/ai-thread-runtime"; +import { resolveWorkspaceAiChatModelId } from "#/features/workspaces/ai/models"; + +import type { WorkspaceAgentOutput } from "./harness"; + +export interface ScoreResult { + score: number; // 0..1 + pass: boolean; + message: string; +} + +/** Every tool call the model made carries valid arguments per its real zod schema. */ +export function scoreToolInputsValid(output: WorkspaceAgentOutput): ScoreResult { + const invalid = output.toolCalls.filter((call) => !call.valid); + const pass = invalid.length === 0; + return { + score: output.toolCalls.length === 0 ? 1 : 1 - invalid.length / output.toolCalls.length, + pass, + message: pass + ? "all tool inputs valid" + : invalid.map((call) => `${call.name}: ${call.issues.join("; ") || "invalid"}`).join(" | "), + }; +} + +/** Model called every expected tool at least once (order-independent). */ +export function scoreExpectedTools( + output: WorkspaceAgentOutput, + expectedTools: string[], +): ScoreResult { + const called = new Set(output.toolCalls.map((call) => call.name)); + const missing = expectedTools.filter((name) => !called.has(name)); + const pass = missing.length === 0; + return { + score: expectedTools.length === 0 ? 1 : 1 - missing.length / expectedTools.length, + pass, + message: pass + ? `called: [${[...called].join(", ")}]` + : `missing: [${missing.join(", ")}] — called: [${[...called].join(", ") || "none"}]`, + }; +} + +/** Model called none of the forbidden tools (e.g. no writes on a read-only turn). */ +export function scoreNoForbiddenTools( + output: WorkspaceAgentOutput, + forbiddenTools: string[], +): ScoreResult { + const forbidden = new Set(forbiddenTools); + const hits = output.toolCalls.map((call) => call.name).filter((name) => forbidden.has(name)); + const pass = hits.length === 0; + return { + score: pass ? 1 : 0, + pass, + message: pass ? "no forbidden tools called" : `forbidden tools called: [${hits.join(", ")}]`, + }; +} + +const QUALITY_VERDICT_SCHEMA = z.object({ + pass: z.boolean(), + score: z.number().min(0).max(1), + reasoning: z.string(), +}); + +// A cheap, low-variance model to grade natural-language answers against a rubric. +const JUDGE_MODEL_ID = resolveWorkspaceAiChatModelId("claude-haiku"); + +/** + * LLM-as-judge: grade a free-text answer against a rubric. Deterministic checks + * (schema/tool choice above) can't judge prose — this can, at the cost of a call. + */ +export async function scoreAnswerQuality(args: { + prompt: string; + answer: string; + rubric: string; +}): Promise { + const result = await generateText({ + model: getWorkspaceAiLanguageModel(JUDGE_MODEL_ID, env, "eval-judge"), + providerOptions: getWorkspaceAiGatewayProviderOptions({ modelId: JUDGE_MODEL_ID }), + output: Output.object({ schema: QUALITY_VERDICT_SCHEMA }), + system: + "You are a strict grader. Score how well the ASSISTANT ANSWER satisfies the RUBRIC for the given USER PROMPT. Return pass=false unless the rubric is clearly met. score is 0..1.", + prompt: `USER PROMPT:\n${args.prompt}\n\nASSISTANT ANSWER:\n${args.answer}\n\nRUBRIC:\n${args.rubric}`, + }); + + const verdict = result.output; + return { + score: verdict?.score ?? 0, + pass: verdict?.pass ?? false, + message: verdict?.reasoning ?? "no verdict returned", + }; +} diff --git a/eval/workspace-tools.eval.ts b/eval/workspace-tools.eval.ts new file mode 100644 index 000000000..c3c693047 --- /dev/null +++ b/eval/workspace-tools.eval.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; + +import { workspaceToolCases } from "./datasets/workspace-tools.cases"; +import { runWorkspaceAgent } from "./support/harness"; +import { + scoreAnswerQuality, + scoreExpectedTools, + scoreNoForbiddenTools, + scoreToolInputsValid, +} from "./support/scorers"; + +// Live evals: real model turns through the real gateway wiring. On-demand and +// billed, so they live outside `pnpm test` — run with `pnpm eval`. Skip cleanly +// when the gateway key is absent instead of failing. +describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("workspace tools", () => { + it.for(workspaceToolCases)("$name", async (testCase) => { + const output = await runWorkspaceAgent(testCase.input); + + // 1. Deterministic — every tool call carries schema-valid arguments. Catches + // a broken `.describe()` or a field the model can no longer fill. + const inputsValid = scoreToolInputsValid(output); + expect(inputsValid.pass, `invalid tool inputs — ${inputsValid.message}`).toBe(true); + + // 2. Deterministic — reached for the expected tools, avoided the forbidden ones. + if (testCase.expectedTools?.length) { + const choice = scoreExpectedTools(output, testCase.expectedTools); + expect(choice.pass, `tool choice — ${choice.message}`).toBe(true); + } + if (testCase.forbiddenTools?.length) { + const forbidden = scoreNoForbiddenTools(output, testCase.forbiddenTools); + expect(forbidden.pass, forbidden.message).toBe(true); + } + + // 3. Model-graded — grade the prose answer against a rubric when set. + if (testCase.qualityRubric) { + const quality = await scoreAnswerQuality({ + prompt: testCase.input.prompt, + answer: output.text, + rubric: testCase.qualityRubric, + }); + expect(quality.pass, `answer quality — ${quality.message}`).toBe(true); + } + }); +}); diff --git a/package.json b/package.json index fe2e28f95..1ecba0ab9 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,8 @@ "prepare": "vp config --no-agent", "test": "vp test --run", "test:workers": "vp test --run --project=workers", + "eval": "infisical run --env=dev --path=/app -- vitest --config vitest.evals.config.ts --run", + "eval:watch": "infisical run --env=dev --path=/app -- vitest --config vitest.evals.config.ts", "deploy:worker": "CLOUDFLARE_ENV=production wrangler deploy", "deploy:worker:staging": "CLOUDFLARE_ENV=staging wrangler deploy --containers-rollout=none", "deploy": "vp run build:production && vp run deploy:worker", @@ -179,7 +181,6 @@ "vite-bundle-analyzer": "^1.3.9", "vite-plus": "^0.2.7", "vitest": "4.1.10", - "vitest-evals": "^0.15.0", "wrangler": "^4.118.0" }, "engines": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 98eda3333..04a41b2e5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -347,7 +347,7 @@ importers: version: 1.50.0(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(wrangler@4.118.0(@cloudflare/workers-types@5.20260801.1)) '@cloudflare/vitest-pool-workers': specifier: ^0.18.8 - version: 0.18.8(@cloudflare/workers-types@5.20260801.1)(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3))) + version: 0.18.8(@cloudflare/workers-types@5.20260801.1)(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10) '@cloudflare/workers-types': specifier: ^5.20260801.1 version: 5.20260801.1 @@ -438,9 +438,6 @@ importers: vitest: specifier: 4.1.10 version: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)) - vitest-evals: - specifier: ^0.15.0 - version: 0.15.0(ai@7.0.48(zod@4.4.3))(tinyrainbow@3.1.1)(vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)))(zod@4.4.3) wrangler: specifier: ^4.118.0 version: 4.118.0(@cloudflare/workers-types@5.20260801.1) @@ -4711,12 +4708,6 @@ packages: babel-plugin-react-compiler: optional: true - '@vitest-evals/core@0.15.0': - resolution: {integrity: sha512-kqyg3ocVtZxr3rP9oh01oAycRgXNRasvTgLZ8MbFGBJ4BRd4tk7Bgww253HY7fw7ZFUp4UPYNmDp5AcHajm++w==} - - '@vitest-evals/report-ui@0.15.0': - resolution: {integrity: sha512-icrutiRhZ2T1MrHo2RW09hcEwfi/tV/FgsGAE76Wd0C02tZyCleQRjt11RWhBJiYDPpsjjjTHGCHIO6tmtX5zQ==} - '@vitest/browser-preview@4.1.10': resolution: {integrity: sha512-14MJrL59ZFkqXLjwfSk6RzTDy5Czf9UG4+8q8L6Gxjs2aPjEce/cVNYV14bXAc2BvMjUNu904+ZEZA1Xc1wtvQ==} peerDependencies: @@ -9273,20 +9264,6 @@ packages: vite: optional: true - vitest-evals@0.15.0: - resolution: {integrity: sha512-bOYALQdLXKYFRCV8aQ+yLeH7XkmYsTxUWQxLHzJdIEcIBx2v/kPgG3Zg6m231jMCphuoM6xH4YQB3+0pDP7LLw==} - hasBin: true - peerDependencies: - ai: '>=4 <7' - tinyrainbow: '>=2 <4' - vitest: 4.1.10 - zod: '>=3 <5' - peerDependenciesMeta: - ai: - optional: true - zod: - optional: true - vitest@4.1.10: resolution: {integrity: sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==} engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -10285,7 +10262,7 @@ snapshots: - bufferutil - utf-8-validate - '@cloudflare/vitest-pool-workers@0.18.8(@cloudflare/workers-types@5.20260801.1)(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)))': + '@cloudflare/vitest-pool-workers@0.18.8(@cloudflare/workers-types@5.20260801.1)(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10)': dependencies: '@vitest/runner': 4.1.10 '@vitest/snapshot': 4.1.10 @@ -13433,14 +13410,6 @@ snapshots: '@rolldown/plugin-babel': 0.2.3(@babel/core@7.29.7)(@babel/runtime@7.29.7)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(rolldown@1.1.2) babel-plugin-react-compiler: 1.0.0 - '@vitest-evals/core@0.15.0': - dependencies: - zod: 4.4.3 - - '@vitest-evals/report-ui@0.15.0': - dependencies: - '@vitest-evals/core': 0.15.0 - '@vitest/browser-preview@4.1.10(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3))(vitest@4.1.10)': dependencies: '@testing-library/dom': 10.4.1 @@ -18573,16 +18542,6 @@ snapshots: optionalDependencies: vite: '@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0)' - vitest-evals@0.15.0(ai@7.0.48(zod@4.4.3))(tinyrainbow@3.1.1)(vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)))(zod@4.4.3): - dependencies: - '@vitest-evals/core': 0.15.0 - '@vitest-evals/report-ui': 0.15.0 - tinyrainbow: 3.1.1 - vitest: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)) - optionalDependencies: - ai: 7.0.48(zod@4.4.3) - zod: 4.4.3 - vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)): dependencies: '@vitest/expect': 4.1.10 diff --git a/src/features/workspaces/operations/__snapshots__/workspace-tool-surface.test.ts.snap b/src/features/workspaces/operations/__snapshots__/workspace-tool-surface.test.ts.snap new file mode 100644 index 000000000..8352173cc --- /dev/null +++ b/src/features/workspaces/operations/__snapshots__/workspace-tool-surface.test.ts.snap @@ -0,0 +1,555 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`workspace tool surface > system prompt is stable 1`] = ` +"# Identity +- You are ThinkEx's workspace assistant. +- Help the user understand, organize, and work in their actual ThinkEx workspace. + +# Workspace Boundaries +- Actual workspace means user-visible ThinkEx content. Private sandbox means assistant-only scratch files. +- Use actual workspace tools to inspect workspace contents; change the workspace only through actual workspace mutation tools. +- Never use private sandbox files as user-visible workspace items. +- Do not claim to have read actual workspace content unless an actual workspace tool returned it. +- Resolve this/it/that/here/above/the page/this file from current-turn context: selected quotes, then active view, then active/open items. Ask briefly before changes if ambiguous. +- Treat workspace relationships as ambient navigation and provenance context. Use them silently to find and understand relevant items; do not present routine relationship maintenance as user-facing work. Mention relationships only when the user asks about them or when one materially affects the answer. +- Web tools read public web content only. + +# Tool Use +- Follow tool descriptions and schemas. +- Whenever you call a user-visible tool, provide a short plain-English title for that tool call. Treat the title as required, not optional. +- Tool titles must be present-progressive activity phrases like 'Reading workspace', 'Researching sources', or 'Updating workspace'. +- Use time_get_current for exact time in UTC or a requested IANA time zone, and time_calculate_relative for exact relative time math; the current turn includes user-local date/time context. + +# Response Style +- Answer directly first. Be clear, specific, and non-redundant. +- Match depth to the task: stay brief for simple questions; explain from first principles when teaching, debugging, comparing options, or recommending a path. +- Treat user claims as hypotheses, not facts. Evaluate them against the available context before agreeing, and challenge weak assumptions directly but respectfully. +- State assumptions, uncertainty, and tradeoffs when they matter. Use examples, steps, or comparisons only when they make the answer easier to act on. +- Do not open with praise, flattery, or generic validation such as 'You're absolutely right', 'Great question', or 'Good catch'. Avoid filler, repeated restatements, and unnecessary summary sections. + +# Output Format +- Format final answers as GitHub-flavored Markdown. Use concise headings, lists, blockquotes, links, tables, task lists, strikethrough, and fenced code blocks with language tags when they improve clarity. +- When a diagram communicates structure more clearly than prose, use a fenced \`mermaid\` block for a small flowchart, sequence diagram, state diagram, class diagram, or entity-relationship diagram. Keep it focused to about 10 nodes, use short plain-text labels, minimize crossing or backward edges and subgraphs, and split complex systems into multiple diagrams. +- Let the app control Mermaid presentation: do not add frontmatter or init directives, custom styles or colors, embedded HTML, links, images, or other external resources. Include a concise \`accTitle\` and \`accDescr\` describing the diagram. +- When writing Markdown with math, use \`$...$\` for inline math and \`$$...$$\` on separate lines for block math. Do not use \`\\(...\\)\` or \`\\[...\\]\`; our renderer only understands dollar-sign delimiters. +- Chemistry renders with \`\\ce{...}\` (e.g. \`$\\ce{CH4 + 2 O2 -> CO2 + 2 H2O}$\`) and quantities with units render with \`\\pu{...}\`. +- CRITICAL: every literal dollar sign — every price, cost, salary, or currency figure — MUST be escaped as \`\\$\`. Write \`\\$5\`, not \`$5\`. Write \`\\$1,000 − \\$200 = \\$800\`, not \`$1,000 − $200 = $800\`. A missed escape turns the price into broken math on screen. + +# Memory +- Use memory only for durable preferences, workspace goals, thread goals, and decisions. Do not store transient requests, secrets, full documents, item bodies, or actual workspace state." +`; + +exports[`workspace tool surface > workspace_create_items input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "items": { + "description": "One or more folders or documents to create in order, at most 20. Parent folders must already exist or be created earlier in the same request.", + "items": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "path": { + "description": "Final absolute path for the folder to create.", + "minLength": 1, + "type": "string", + }, + "relations": { + "description": "Optional relationships from this new folder to other workspace items, at most 20.", + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "description": "\`derived_from\` means this item was created or materially changed from the linked item. \`references\` means this item cites or points to the linked item.", + "enum": [ + "derived_from", + "references", + ], + "type": "string", + }, + "note": { + "description": "Optional short source detail, like pages 12-14 or section on photosynthesis.", + "maxLength": 240, + "type": "string", + }, + "path": { + "description": "Absolute path of the related ThinkEx workspace item.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "kind", + "path", + ], + "type": "object", + }, + "maxItems": 20, + "type": "array", + }, + "type": { + "const": "folder", + "type": "string", + }, + }, + "required": [ + "type", + "path", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "initialContent": { + "description": "Optional initial HTML content for the document.", + "maxLength": 512000, + "type": "string", + }, + "path": { + "description": "Final absolute path for the document to create.", + "minLength": 1, + "type": "string", + }, + "relations": { + "description": "Optional relationships from this new document to other workspace items, at most 20.", + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "description": "\`derived_from\` means this item was created or materially changed from the linked item. \`references\` means this item cites or points to the linked item.", + "enum": [ + "derived_from", + "references", + ], + "type": "string", + }, + "note": { + "description": "Optional short source detail, like pages 12-14 or section on photosynthesis.", + "maxLength": 240, + "type": "string", + }, + "path": { + "description": "Absolute path of the related ThinkEx workspace item.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "kind", + "path", + ], + "type": "object", + }, + "maxItems": 20, + "type": "array", + }, + "type": { + "const": "document", + "type": "string", + }, + }, + "required": [ + "type", + "path", + ], + "type": "object", + }, + ], + }, + "maxItems": 20, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "items", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_delete_items input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "paths": { + "description": "Absolute paths of one or more actual ThinkEx workspace items to delete, at most 20.", + "items": { + "minLength": 1, + "type": "string", + }, + "maxItems": 20, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "paths", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_edit_item input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "edits": { + "description": "Ordered structural HTML edits, at most 40. For targeted operations, copy data-ref into the "ref" field; there is no "target" field. These refs are local to this document and are not workspace citation refs.", + "items": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "html": { + "description": "Schema-constrained HTML fragment. Model-supplied data-ref attributes are ignored.", + "maxLength": 512000, + "type": "string", + }, + "op": { + "enum": [ + "insert_after", + "insert_before", + "replace", + ], + "type": "string", + }, + "ref": { + "description": "Exact data-ref from a recent HTML read. Put it in the "ref" field, never "target".", + "maxLength": 64, + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "html", + "op", + "ref", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "op": { + "const": "delete", + "type": "string", + }, + "ref": { + "description": "Exact data-ref from a recent HTML read. Put it in the "ref" field, never "target".", + "maxLength": 64, + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "op", + "ref", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "html": { + "description": "Schema-constrained HTML fragment. Model-supplied data-ref attributes are ignored.", + "maxLength": 512000, + "type": "string", + }, + "op": { + "const": "replace_all", + "type": "string", + }, + }, + "required": [ + "html", + "op", + ], + "type": "object", + }, + ], + }, + "maxItems": 40, + "minItems": 1, + "type": "array", + }, + "path": { + "description": "Absolute path of one actual ThinkEx workspace item to edit.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "path", + "edits", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_link_items input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "path": { + "description": "Absolute path of the workspace item to link from.", + "minLength": 1, + "type": "string", + }, + "relations": { + "description": "Relationships from this item to other workspace items, at most 20.", + "items": { + "additionalProperties": false, + "properties": { + "kind": { + "description": "\`derived_from\` means this item was created or materially changed from the linked item. \`references\` means this item cites or points to the linked item.", + "enum": [ + "derived_from", + "references", + ], + "type": "string", + }, + "note": { + "description": "Optional short source detail, like pages 12-14 or section on photosynthesis.", + "maxLength": 240, + "type": "string", + }, + "path": { + "description": "Absolute path of the related ThinkEx workspace item.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "kind", + "path", + ], + "type": "object", + }, + "maxItems": 20, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "path", + "relations", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_list_items input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "offset": { + "description": "Zero-based item offset. Use nextOffset from the previous result to continue.", + "maximum": 9007199254740991, + "minimum": 0, + "type": "integer", + }, + "path": { + "description": "Absolute path in the actual ThinkEx workspace. Defaults to /.", + "minLength": 1, + "type": "string", + }, + "recursive": { + "description": "Include nested descendants. Defaults to false for immediate children only.", + "type": "boolean", + }, + }, + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_move_items input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "destinationPath": { + "description": "Absolute path of the destination folder. Use / for the workspace root.", + "minLength": 1, + "type": "string", + }, + "paths": { + "description": "Absolute paths of one or more actual ThinkEx workspace items to move, at most 20.", + "items": { + "minLength": 1, + "type": "string", + }, + "maxItems": 20, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "destinationPath", + "paths", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_read_items input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "requests": { + "description": "Ordered workspace content reads, at most 20.", + "items": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "mode": { + "const": "start", + "type": "string", + }, + "path": { + "description": "Absolute path of the workspace item to read.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "path", + "mode", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "mode": { + "const": "pages", + "type": "string", + }, + "path": { + "description": "Absolute path of the workspace item to read.", + "minLength": 1, + "type": "string", + }, + "range": { + "description": "Up to 20 physical pages from an extracted file, like 1, 3, 5-7, or 1,4-6. Defaults to page 1.", + "minLength": 1, + "pattern": "^\\d+(?:\\s*-\\s*\\d+)?(?:\\s*,\\s*\\d+(?:\\s*-\\s*\\d+)?)*$", + "type": "string", + }, + }, + "required": [ + "path", + "mode", + "range", + ], + "type": "object", + }, + { + "additionalProperties": false, + "properties": { + "cursor": { + "description": "Opaque cursor returned by a previous read.", + "maxLength": 4096, + "minLength": 1, + "type": "string", + }, + "mode": { + "const": "continue", + "type": "string", + }, + "path": { + "description": "Absolute path of the workspace item to read.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "path", + "cursor", + "mode", + ], + "type": "object", + }, + ], + }, + "maxItems": 20, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "requests", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_rename_item input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "name": { + "description": "New user-visible item name.", + "maxLength": 160, + "minLength": 1, + "type": "string", + }, + "path": { + "description": "Absolute path of one actual ThinkEx workspace item to rename.", + "minLength": 1, + "type": "string", + }, + }, + "required": [ + "name", + "path", + ], + "type": "object", +} +`; + +exports[`workspace tool surface > workspace_search input schema is stable 1`] = ` +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "path": { + "description": "Optional absolute workspace path. A folder searches recursively; an item searches only that item. Defaults to /.", + "minLength": 1, + "type": "string", + }, + "query": { + "description": "Text or natural-language question to search for in the workspace.", + "maxLength": 1000, + "minLength": 2, + "type": "string", + }, + "types": { + "description": "Optional content types to include. Defaults to documents and files.", + "items": { + "enum": [ + "document", + "file", + ], + "type": "string", + }, + "minItems": 1, + "type": "array", + }, + }, + "required": [ + "query", + ], + "type": "object", +} +`; diff --git a/src/features/workspaces/operations/create-items.ts b/src/features/workspaces/operations/create-items.ts index 9c87ea7d1..671b3204d 100644 --- a/src/features/workspaces/operations/create-items.ts +++ b/src/features/workspaces/operations/create-items.ts @@ -2,8 +2,8 @@ import { getAuthorizedWorkspaceKernel } from "#/features/workspaces/operations/w import { resolveWorkspaceRelations, type WorkspaceRelationInput, - workspaceRelationFailureCodes, } from "#/features/workspaces/operations/relations"; +import { createWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/workspace-operation-failure-codes"; import type { WorkspaceAccessContext } from "#/features/workspaces/operations/workspace-access-context"; import type { WorkspaceKernelPathResolution } from "#/features/workspaces/kernel/workspace-kernel-types"; import { parseDocumentAiHtml } from "#/features/workspaces/documents/document-ai-html"; @@ -32,17 +32,6 @@ export interface CreateWorkspaceItemsOperationInput { items: CreateWorkspaceItemOperationInput[]; } -export const createWorkspaceItemsFailureCodes = [ - "cannot_create_root", - "invalid_initial_content", - "path_already_exists", - "path_not_absolute", - "path_not_canonical", - "path_not_folder", - "path_not_found", - ...workspaceRelationFailureCodes, -] as const; - type CreateWorkspaceItemsFailureCode = (typeof createWorkspaceItemsFailureCodes)[number]; export interface CreateWorkspaceItemsFailure { diff --git a/src/features/workspaces/operations/delete-items.ts b/src/features/workspaces/operations/delete-items.ts index 4064db94a..7ac7e7fe0 100644 --- a/src/features/workspaces/operations/delete-items.ts +++ b/src/features/workspaces/operations/delete-items.ts @@ -9,11 +9,7 @@ export interface DeleteWorkspaceItemsOperationInput { paths: string[]; } -export const deleteWorkspaceItemsFailureCodes = [ - "cannot_delete_root", - "path_not_absolute", - "path_not_found", -] as const; +import { deleteWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/workspace-operation-failure-codes"; export interface DeleteWorkspaceItemsFailure { code: (typeof deleteWorkspaceItemsFailureCodes)[number]; diff --git a/src/features/workspaces/operations/edit-item.ts b/src/features/workspaces/operations/edit-item.ts index b3029eca4..7613e7942 100644 --- a/src/features/workspaces/operations/edit-item.ts +++ b/src/features/workspaces/operations/edit-item.ts @@ -4,23 +4,11 @@ import { resolveWorkspaceExistingItemPath, } from "#/features/workspaces/operations/workspace-operation-context"; import type { WorkspaceAccessContext } from "#/features/workspaces/operations/workspace-access-context"; -import { - type DocumentAiEdit, - documentAiEditFailureCodes, -} from "#/features/workspaces/documents/document-ai-edits"; +import { type DocumentAiEdit } from "#/features/workspaces/documents/document-ai-edits"; +import { editWorkspaceItemFailureCodes } from "#/features/workspaces/operations/workspace-operation-failure-codes"; import type { DocumentEditLineChanges } from "#/features/workspaces/documents/document-edit-receipt"; import { resolveDocumentCitations } from "#/features/workspaces/operations/document-citations"; -export const editWorkspaceItemFailureCodes = [ - "cannot_edit_root", - "path_not_absolute", - "path_not_found", - "unsupported_item_type", - ...documentAiEditFailureCodes, - "content_changed", - "operation_id_conflict", -] as const; - type EditWorkspaceItemFailureCode = (typeof editWorkspaceItemFailureCodes)[number]; export interface EditWorkspaceItemOperationInput { diff --git a/src/features/workspaces/operations/link-items.ts b/src/features/workspaces/operations/link-items.ts index 6e8e42e04..087842229 100644 --- a/src/features/workspaces/operations/link-items.ts +++ b/src/features/workspaces/operations/link-items.ts @@ -2,8 +2,8 @@ import type { WorkspaceItemSummary } from "#/features/workspaces/contracts"; import { resolveWorkspaceRelations, type WorkspaceRelationInput, - workspaceRelationFailureCodes, } from "#/features/workspaces/operations/relations"; +import { linkWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/workspace-operation-failure-codes"; import type { WorkspaceAccessContext } from "#/features/workspaces/operations/workspace-access-context"; import { getAuthorizedWorkspaceKernel, @@ -15,13 +15,6 @@ export interface LinkWorkspaceItemsOperationInput { relations: WorkspaceRelationInput[]; } -export const linkWorkspaceItemsFailureCodes = [ - "cannot_link_root", - "path_not_absolute", - "path_not_found", - ...workspaceRelationFailureCodes, -] as const; - type LinkWorkspaceItemsFailureCode = (typeof linkWorkspaceItemsFailureCodes)[number]; interface LinkWorkspaceItemsFailure { diff --git a/src/features/workspaces/operations/move-items.ts b/src/features/workspaces/operations/move-items.ts index a5fa7aff2..840c59ef0 100644 --- a/src/features/workspaces/operations/move-items.ts +++ b/src/features/workspaces/operations/move-items.ts @@ -15,17 +15,7 @@ export interface MoveWorkspaceItemsOperationInput { paths: string[]; } -export const moveWorkspaceItemsFailureCodes = [ - "already_in_destination", - "cannot_move_into_descendant", - "cannot_move_root", - "destination_path_not_absolute", - "destination_path_not_folder", - "destination_path_not_found", - "path_already_exists", - "path_not_absolute", - "path_not_found", -] as const; +import { moveWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/workspace-operation-failure-codes"; interface MoveWorkspaceDestinationFailure { code: diff --git a/src/features/workspaces/operations/rename-item.ts b/src/features/workspaces/operations/rename-item.ts index ddffd801f..79cd9f784 100644 --- a/src/features/workspaces/operations/rename-item.ts +++ b/src/features/workspaces/operations/rename-item.ts @@ -14,12 +14,7 @@ export interface RenameWorkspaceItemOperationInput { path: string; } -export const renameWorkspaceItemFailureCodes = [ - "cannot_rename_root", - "path_already_exists", - "path_not_absolute", - "path_not_found", -] as const; +import { renameWorkspaceItemFailureCodes } from "#/features/workspaces/operations/workspace-operation-failure-codes"; export interface RenameWorkspaceItemFailure { code: (typeof renameWorkspaceItemFailureCodes)[number]; diff --git a/src/features/workspaces/operations/workspace-operation-failure-codes.ts b/src/features/workspaces/operations/workspace-operation-failure-codes.ts new file mode 100644 index 000000000..6294c9d7b --- /dev/null +++ b/src/features/workspaces/operations/workspace-operation-failure-codes.ts @@ -0,0 +1,60 @@ +import { documentAiEditFailureCodes } from "#/features/workspaces/documents/document-ai-edits"; +import { workspaceRelationFailureCodes } from "#/features/workspaces/operations/relations"; + +// The failure-code vocabulary for each workspace operation. This is a pure leaf +// (no operation impls, no kernel) so the model-facing schema + contract layer can +// depend on it without dragging the worker runtime — which also lets these run in +// plain Node evals. Impls import their codes from here rather than owning them. + +export const createWorkspaceItemsFailureCodes = [ + "cannot_create_root", + "invalid_initial_content", + "path_already_exists", + "path_not_absolute", + "path_not_canonical", + "path_not_folder", + "path_not_found", + ...workspaceRelationFailureCodes, +] as const; + +export const deleteWorkspaceItemsFailureCodes = [ + "cannot_delete_root", + "path_not_absolute", + "path_not_found", +] as const; + +export const editWorkspaceItemFailureCodes = [ + "cannot_edit_root", + "path_not_absolute", + "path_not_found", + "unsupported_item_type", + ...documentAiEditFailureCodes, + "content_changed", + "operation_id_conflict", +] as const; + +export const linkWorkspaceItemsFailureCodes = [ + "cannot_link_root", + "path_not_absolute", + "path_not_found", + ...workspaceRelationFailureCodes, +] as const; + +export const moveWorkspaceItemsFailureCodes = [ + "already_in_destination", + "cannot_move_into_descendant", + "cannot_move_root", + "destination_path_not_absolute", + "destination_path_not_folder", + "destination_path_not_found", + "path_already_exists", + "path_not_absolute", + "path_not_found", +] as const; + +export const renameWorkspaceItemFailureCodes = [ + "cannot_rename_root", + "path_already_exists", + "path_not_absolute", + "path_not_found", +] as const; diff --git a/src/features/workspaces/operations/workspace-tool-schemas.ts b/src/features/workspaces/operations/workspace-tool-schemas.ts index 5e410d3f7..26d3a91fb 100644 --- a/src/features/workspaces/operations/workspace-tool-schemas.ts +++ b/src/features/workspaces/operations/workspace-tool-schemas.ts @@ -4,12 +4,14 @@ import { workspaceReadItemsInputSchema, workspaceReadItemsOutputSchema, } from "#/features/workspaces/content/workspace-content-contract"; -import { createWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/create-items"; -import { deleteWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/delete-items"; -import { editWorkspaceItemFailureCodes } from "#/features/workspaces/operations/edit-item"; -import { linkWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/link-items"; -import { moveWorkspaceItemsFailureCodes } from "#/features/workspaces/operations/move-items"; -import { renameWorkspaceItemFailureCodes } from "#/features/workspaces/operations/rename-item"; +import { + createWorkspaceItemsFailureCodes, + deleteWorkspaceItemsFailureCodes, + editWorkspaceItemFailureCodes, + linkWorkspaceItemsFailureCodes, + moveWorkspaceItemsFailureCodes, + renameWorkspaceItemFailureCodes, +} from "#/features/workspaces/operations/workspace-operation-failure-codes"; import { workspaceItemTypeSchema, workspaceRelationKindSchema, diff --git a/src/features/workspaces/operations/workspace-tool-surface.test.ts b/src/features/workspaces/operations/workspace-tool-surface.test.ts new file mode 100644 index 000000000..b24605785 --- /dev/null +++ b/src/features/workspaces/operations/workspace-tool-surface.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { getAIThreadSoulPrompt } from "#/features/workspaces/ai/ai-thread-soul-prompt"; +import { + workspaceCreateItemsInputSchema, + workspaceDeleteItemsInputSchema, + workspaceEditItemInputSchema, + workspaceLinkItemsInputSchema, + workspaceListItemsInputSchema, + workspaceMoveItemsInputSchema, + workspaceReadItemsInputSchema, + workspaceRenameItemInputSchema, + workspaceSearchInputSchema, +} from "#/features/workspaces/operations/workspace-tool-schemas"; + +// Free, deterministic regression net for the model-facing surface: the assembled +// system prompt and each tool's input JSON schema (field `.describe()` text +// included). A change to a schema shape, a field description, or the soul prompt +// shows up here as a reviewable diff — zero model calls, runs in normal CI. +// +// Update intentionally with `pnpm test -u` when the change is deliberate; an +// unexpected diff is a regression to look at. Whether the model actually *uses* +// the surface correctly is the live evals' job: `pnpm eval`. + +const TOOL_INPUT_SCHEMAS = { + workspace_list_items: workspaceListItemsInputSchema, + workspace_read_items: workspaceReadItemsInputSchema, + workspace_search: workspaceSearchInputSchema, + workspace_create_items: workspaceCreateItemsInputSchema, + workspace_edit_item: workspaceEditItemInputSchema, + workspace_delete_items: workspaceDeleteItemsInputSchema, + workspace_move_items: workspaceMoveItemsInputSchema, + workspace_rename_item: workspaceRenameItemInputSchema, + workspace_link_items: workspaceLinkItemsInputSchema, +} as const; + +describe("workspace tool surface", () => { + it("system prompt is stable", () => { + expect(getAIThreadSoulPrompt()).toMatchSnapshot(); + }); + + for (const [name, schema] of Object.entries(TOOL_INPUT_SCHEMAS)) { + it(`${name} input schema is stable`, () => { + expect(z.toJSONSchema(schema)).toMatchSnapshot(); + }); + } +}); diff --git a/tsconfig.json b/tsconfig.json index 36eb548e9..a62ad5b0a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,9 +3,11 @@ "src/**/*.ts", "src/**/*.tsx", "test/**/*.ts", + "eval/**/*.ts", "drizzle.config.ts", "vite.config.ts", - "vitest.config.ts" + "vitest.config.ts", + "vitest.evals.config.ts" ], "compilerOptions": { "target": "ES2022", diff --git a/vitest.evals.config.ts b/vitest.evals.config.ts new file mode 100644 index 000000000..97ee23683 --- /dev/null +++ b/vitest.evals.config.ts @@ -0,0 +1,57 @@ +import { fileURLToPath } from "node:url"; + +import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; +import agents from "agents/vite"; +import { defineConfig } from "vite-plus/test/config"; + +// Evals exercise the real workspace tool surface (schemas + descriptions) and the +// real gateway model wiring. That code is worker-runtime code — it imports the +// kernel, Durable Objects, and `cloudflare:workers` — so, exactly like the app's +// own `*.worker.test.ts`, evals run in the Cloudflare workers pool rather than +// Node. This gives us the real modules with no stubbing. +// +// They live in their own config so they never run in the normal `pnpm test` +// pipeline: they hit real models (real latency + spend). Run them on demand with +// `pnpm eval`, which injects AI_GATEWAY_API_KEY (via Infisical) into the pool. +export default defineConfig({ + test: { + projects: [ + { + resolve: { + tsconfigPaths: true, + alias: { + // The worker entry pulls in the TanStack Start server handler, whose + // build-time virtual specifiers don't resolve under Vitest. The app's + // own workers project stubs it the same way. + "@tanstack/react-start/server": fileURLToPath( + new URL("./test/stubs/tanstack-start-server.ts", import.meta.url), + ), + }, + }, + plugins: [ + // Agent classes use TC39 decorators that Oxc can't transform yet; this + // plugin lowers them (same as the app build and workers test project). + agents(), + cloudflareTest({ + main: "./test/worker-entry.ts", + remoteBindings: false, + wrangler: { configPath: "./wrangler.jsonc" }, + // AI_GATEWAY_API_KEY is a secret, so it isn't in wrangler vars. Inject + // it from the process env (Infisical supplies it under `pnpm eval`) so + // the real gateway wiring can authenticate. + miniflare: { + bindings: { AI_GATEWAY_API_KEY: process.env.AI_GATEWAY_API_KEY ?? "" }, + }, + }), + ], + test: { + name: "evals", + include: ["eval/**/*.eval.ts"], + // Real model turns are slow; give them room. + testTimeout: 120_000, + hookTimeout: 120_000, + }, + }, + ], + }, +}); From 3193b7b871105ab8848f25517f38d19db9a7aaf1 Mon Sep 17 00:00:00 2001 From: Urjit Chakraborty <135136842+urjitc@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:27:47 -0400 Subject: [PATCH 5/5] fix(eval): exercise the real tool surface in the workspace agent eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eval hand-rolled its toolset instead of the production path, so it diverged in three ways the reviewers caught: - Sent raw Zod schemas (with `maxItems`, which Anthropic-compatible providers reject) instead of the provider-compatible schema. Now reuses the exported `createProviderCompatibleInputSchema`. - Omitted `inputExamples`, which the gateway middleware injects into the prompt in production. Now passed through per tool. - Gave `workspace_read_items` the same neutral `{ok:true}` stub as every tool, so the read→edit case could pass with a fabricated ref or a whole-doc `replace_all`. Now returns a fixture with real document HTML + `data-ref`s, and `scoreTargetedEditProvenance` requires a targeted edit whose ref traces back to that fixture. Also collapses a two-pass map().filter() in scoreNoForbiddenTools. --- eval/datasets/workspace-tools.cases.ts | 3 ++ eval/support/harness.ts | 45 ++++++++++++++++---- eval/support/scorers.ts | 40 ++++++++++++++++- eval/workspace-tools.eval.ts | 10 ++++- src/features/workspaces/ai/ai-thread-tool.ts | 2 +- 5 files changed, 89 insertions(+), 11 deletions(-) diff --git a/eval/datasets/workspace-tools.cases.ts b/eval/datasets/workspace-tools.cases.ts index fac028d82..1dec989a1 100644 --- a/eval/datasets/workspace-tools.cases.ts +++ b/eval/datasets/workspace-tools.cases.ts @@ -9,6 +9,8 @@ export interface WorkspaceToolCase { forbiddenTools?: string[]; /** When set, the final answer is graded by the LLM judge against this rubric. */ qualityRubric?: string; + /** When true, the turn must produce a targeted edit whose ref traces to the read fixture. */ + requiresTargetedEditFromRead?: boolean; } // A view-only scope block: mirrors what beforeTurn injects for a read-only viewer. @@ -46,6 +48,7 @@ export const workspaceToolCases: WorkspaceToolCase[] = [ "In /Notes/Standup.md, add a second bullet that says 'Review metrics'. Read the document first, then make the edit.", }, expectedTools: ["workspace_read_items", "workspace_edit_item"], + requiresTargetedEditFromRead: true, }, { name: "respects a read-only turn", diff --git a/eval/support/harness.ts b/eval/support/harness.ts index 25866449e..b4eb3fe0d 100644 --- a/eval/support/harness.ts +++ b/eval/support/harness.ts @@ -3,6 +3,7 @@ import { asSchema, generateText, stepCountIs, tool, type ToolSet } from "ai"; import type { z } from "zod"; import { getAIThreadSoulPrompt } from "#/features/workspaces/ai/ai-thread-soul-prompt"; +import { createProviderCompatibleInputSchema } from "#/features/workspaces/ai/ai-thread-tool"; import { getWorkspaceAiGatewayProviderOptions, getWorkspaceAiLanguageModel, @@ -44,20 +45,48 @@ export interface WorkspaceAgentInput { system?: string; } -// Real workspace tools, but with stubbed execution. Evals grade tool *selection* -// and *argument validity* — the two things schema/instruction changes move — so we -// never touch a Durable Object or mutate a real workspace. +// Deterministic read fixture: document HTML carrying real `data-ref` values, so a +// read→edit turn can produce a *targeted* edit whose ref traces back to the read. +// `scoreTargetedEditProvenance` checks that provenance against these refs. +const STANDUP_HEADING_REF = "b_standupHead1.r_head000001"; +const STANDUP_LIST_REF = "b_standupList1.r_bullet0001"; +export const EVAL_READ_FIXTURE_REFS = [STANDUP_HEADING_REF, STANDUP_LIST_REF]; + +// Per-tool stubbed outputs. Reads return editable HTML + refs; everything else +// returns a neutral success so a follow-up step can still proceed. No real +// Durable Object is touched and no workspace is mutated. +const EVAL_TOOL_FIXTURES: Record = { + workspace_read_items: { + items: [ + { + path: "/Notes/Standup.md", + type: "document", + html: `

Standup

  • Discuss roadmap
`, + }, + ], + }, +}; + +function evalToolFixture(toolName: string): unknown { + return EVAL_TOOL_FIXTURES[toolName] ?? { ok: true, note: "eval stub — no real mutation" }; +} + +// Real workspace tools with stubbed execution. Evals grade tool *selection* and +// *argument validity*, so the model must see the SAME surface production sends: +// the provider-compatible schema (maxItems stripped, which Anthropic requires) via +// the shared `createProviderCompatibleInputSchema`, plus the `inputExamples` the +// gateway middleware injects. Only execution is stubbed. function buildEvalToolSet(): ToolSet { return Object.fromEntries( workspaceToolDefinitions.map((definition) => [ definition.name, tool({ description: definition.description, - // Heterogeneous across tools; the eval only needs the schema rendered - // to the model, so widen past the per-tool union. - inputSchema: asSchema(definition.inputSchema as z.ZodTypeAny), - // Neutral stub so a follow-up step (e.g. read → edit) can still proceed. - execute: async () => ({ ok: true, note: "eval stub — no real mutation" }), + inputSchema: createProviderCompatibleInputSchema( + asSchema(definition.inputSchema as z.ZodTypeAny), + ), + inputExamples: definition.inputExamples, + execute: async () => evalToolFixture(definition.name), }), ]), ) as ToolSet; diff --git a/eval/support/scorers.ts b/eval/support/scorers.ts index 93ff0dff9..17af85dd7 100644 --- a/eval/support/scorers.ts +++ b/eval/support/scorers.ts @@ -52,7 +52,10 @@ export function scoreNoForbiddenTools( forbiddenTools: string[], ): ScoreResult { const forbidden = new Set(forbiddenTools); - const hits = output.toolCalls.map((call) => call.name).filter((name) => forbidden.has(name)); + const hits: string[] = []; + for (const call of output.toolCalls) { + if (forbidden.has(call.name)) hits.push(call.name); + } const pass = hits.length === 0; return { score: pass ? 1 : 0, @@ -61,6 +64,41 @@ export function scoreNoForbiddenTools( }; } +/** + * For a read→edit turn: the model must submit a *targeted* edit (replace / insert / + * delete) whose `ref` came from the read fixture — not a fabricated ref, and not a + * whole-document `replace_all`. Without this, a hollow read stub plus the permissive + * `ref` schema (any nonempty string) let a hallucinated target score a false pass. + */ +export function scoreTargetedEditProvenance( + output: WorkspaceAgentOutput, + validRefs: string[], +): ScoreResult { + const refs = new Set(validRefs); + const edits = output.toolCalls + .filter((call) => call.name === "workspace_edit_item") + .flatMap((call) => { + const input = call.input as { edits?: Array<{ op?: string; ref?: string }> }; + return Array.isArray(input.edits) ? input.edits : []; + }); + const isValidRef = (edit: { ref?: string }) => typeof edit.ref === "string" && refs.has(edit.ref); + const targeted = edits.filter((edit) => edit.op !== "replace_all" && isValidRef(edit)); + const fabricated = edits.filter((edit) => edit.op !== "replace_all" && !isValidRef(edit)); + const usedReplaceAll = edits.some((edit) => edit.op === "replace_all"); + const pass = targeted.length > 0 && fabricated.length === 0 && !usedReplaceAll; + + const reasons: string[] = []; + if (targeted.length === 0) reasons.push("no targeted edit used a ref from the read"); + if (fabricated.length > 0) + reasons.push(`fabricated ref(s): ${fabricated.map((e) => e.ref ?? "").join(", ")}`); + if (usedReplaceAll) reasons.push("used replace_all instead of a targeted edit"); + return { + score: pass ? 1 : 0, + pass, + message: pass ? "targeted edit used a ref from the read fixture" : reasons.join("; "), + }; +} + const QUALITY_VERDICT_SCHEMA = z.object({ pass: z.boolean(), score: z.number().min(0).max(1), diff --git a/eval/workspace-tools.eval.ts b/eval/workspace-tools.eval.ts index c3c693047..cf3c7e274 100644 --- a/eval/workspace-tools.eval.ts +++ b/eval/workspace-tools.eval.ts @@ -1,11 +1,12 @@ import { describe, expect, it } from "vitest"; import { workspaceToolCases } from "./datasets/workspace-tools.cases"; -import { runWorkspaceAgent } from "./support/harness"; +import { EVAL_READ_FIXTURE_REFS, runWorkspaceAgent } from "./support/harness"; import { scoreAnswerQuality, scoreExpectedTools, scoreNoForbiddenTools, + scoreTargetedEditProvenance, scoreToolInputsValid, } from "./support/scorers"; @@ -31,6 +32,13 @@ describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("workspace tools", () => { expect(forbidden.pass, forbidden.message).toBe(true); } + // 2b. Deterministic — a read→edit turn made a targeted edit whose ref came + // from the read fixture, not a fabricated ref or a whole-doc replace_all. + if (testCase.requiresTargetedEditFromRead) { + const provenance = scoreTargetedEditProvenance(output, EVAL_READ_FIXTURE_REFS); + expect(provenance.pass, `edit provenance — ${provenance.message}`).toBe(true); + } + // 3. Model-graded — grade the prose answer against a rubric when set. if (testCase.qualityRubric) { const quality = await scoreAnswerQuality({ diff --git a/src/features/workspaces/ai/ai-thread-tool.ts b/src/features/workspaces/ai/ai-thread-tool.ts index a60e4f7bf..3ec7e38b5 100644 --- a/src/features/workspaces/ai/ai-thread-tool.ts +++ b/src/features/workspaces/ai/ai-thread-tool.ts @@ -114,7 +114,7 @@ type ModelJsonSchema = Awaited; * validation still uses the original schema, so omitting this model-facing * hint improves portability without changing accepted application input. */ -function createProviderCompatibleInputSchema(schema: Schema): Schema { +export function createProviderCompatibleInputSchema(schema: Schema): Schema { return jsonSchema( async () => JSON.parse(