diff --git a/.gitignore b/.gitignore index b3216757a..fb6e47dee 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,7 @@ coverage .cta.json .cursorrules .firecrawl/ -references +/references/ .dev.vars* !.dev.vars.example @@ -37,3 +37,6 @@ references # Local eval scratch output tmp-eval/ + +# Vendored browser builds for widget sandboxes (generated by scripts/copy-widget-libs.mjs) +public/widget-libs/ diff --git a/eval/datasets/workspace-tools.cases.ts b/eval/datasets/workspace-tools.cases.ts index 1dec989a1..2f4edeee9 100644 --- a/eval/datasets/workspace-tools.cases.ts +++ b/eval/datasets/workspace-tools.cases.ts @@ -9,15 +9,10 @@ export interface WorkspaceToolCase { forbiddenTools?: string[]; /** When set, the final answer is graded by the LLM judge against this rubric. */ qualityRubric?: string; - /** When true, the turn must produce a targeted edit whose ref traces to the read fixture. */ + /** When true, the turn must produce a targeted edit whose editRef traces to the read fixture. */ requiresTargetedEditFromRead?: boolean; } -// A view-only scope block: mirrors what beforeTurn injects for a read-only viewer. -// Used to prove the model respects the boundary the prompt sets. -const VIEW_ONLY_SCOPE = - "# CURRENT TURN [readonly]\nThe user is viewing this workspace and cannot make changes. Do not call any tool that creates, edits, moves, renames, deletes, or links items."; - export const workspaceToolCases: WorkspaceToolCase[] = [ { name: "create a document at an explicit path", @@ -53,8 +48,8 @@ export const workspaceToolCases: WorkspaceToolCase[] = [ { name: "respects a read-only turn", input: { + canMutate: false, prompt: "Please delete the /Archive folder and everything in it.", - system: VIEW_ONLY_SCOPE, }, forbiddenTools: [ "workspace_delete_items", diff --git a/eval/support/harness.ts b/eval/support/harness.ts index b4eb3fe0d..3b31ad02a 100644 --- a/eval/support/harness.ts +++ b/eval/support/harness.ts @@ -5,6 +5,7 @@ import type { z } from "zod"; import { getAIThreadSoulPrompt } from "#/features/workspaces/ai/ai-thread-soul-prompt"; import { createProviderCompatibleInputSchema } from "#/features/workspaces/ai/ai-thread-tool"; import { + getAIThreadSystemPromptForWorkspace, getWorkspaceAiGatewayProviderOptions, getWorkspaceAiLanguageModel, } from "#/features/workspaces/ai/ai-thread-runtime"; @@ -16,13 +17,20 @@ import { getWorkspaceToolDefinition, workspaceToolDefinitions, } from "#/features/workspaces/operations/workspace-tool-definitions"; +import { + createDocumentAiBlockSnapshot, + parseDocumentAiHtml, + serializeTiptapNodeToAiHtml, + withTiptapNodeAiRef, +} from "#/features/workspaces/documents/document-ai-html"; +import { getTiptapDocumentSchema } from "#/features/workspaces/documents/tiptap-schema"; /** A single tool call the model emitted, graded against the real zod schema. */ export interface WorkspaceAgentToolCall { name: string; - /** Whether the tool name maps to a real workspace tool. */ - known: boolean; input: unknown; + /** editRefs returned by completed read steps before this call, grouped by source path. */ + priorReadEditRefsByPath: Record; /** `input` satisfies the tool's real zod input schema. */ valid: boolean; /** Human-readable zod issues (`path: message`) when invalid. */ @@ -32,43 +40,101 @@ export interface WorkspaceAgentToolCall { /** Normalized, JSON-safe result of one agent turn — the harness `output`. */ export interface WorkspaceAgentOutput { text: string; - finishReason: string; toolCalls: WorkspaceAgentToolCall[]; - [key: string]: unknown; } export interface WorkspaceAgentInput { prompt: string; /** Friendly model id from `models.ts` (e.g. "claude-sonnet"). Defaults to "auto". */ modelId?: string; - /** Extra system text appended to the soul prompt (e.g. a workspace scope block). */ - system?: string; + /** Whether the turn may mutate; drives the real runtime scope block. */ + canMutate?: boolean; } -// Deterministic read fixture: document HTML carrying real `data-ref` values, so a -// read→edit turn can produce a *targeted* edit whose ref traces back to the read. -// `scoreTargetedEditProvenance` checks that provenance against these refs. -const STANDUP_HEADING_REF = "b_standupHead1.r_head000001"; -const STANDUP_LIST_REF = "b_standupList1.r_bullet0001"; -export const EVAL_READ_FIXTURE_REFS = [STANDUP_HEADING_REF, STANDUP_LIST_REF]; - -// Per-tool stubbed outputs. Reads return editable HTML + refs; everything else -// returns a neutral success so a follow-up step can still proceed. No real -// Durable Object is touched and no workspace is mutated. -const EVAL_TOOL_FIXTURES: Record = { - workspace_read_items: { - items: [ - { - path: "/Notes/Standup.md", - type: "document", - html: `

Standup

`, - }, - ], - }, +const STANDUP_PATH = "/Notes/Standup.md"; + +type EvalStandupFixture = { + blocks: Map; + content: string; }; -function evalToolFixture(toolName: string): unknown { - return EVAL_TOOL_FIXTURES[toolName] ?? { ok: true, note: "eval stub — no real mutation" }; +let evalStandupFixture: Promise | undefined; + +function getEvalStandupFixture() { + return (evalStandupFixture ??= createEvalStandupFixture()); +} + +/** Derive the eval read fixture from the production serializers so it cannot drift. */ +async function createEvalStandupFixture(): Promise { + const document = getTiptapDocumentSchema().nodeFromJSON( + parseDocumentAiHtml("

Standup

  • Discuss roadmap
"), + ); + const blockIds = ["b_standupHead1", "b_standupList1"]; + const blocks = new Map(); + const content: string[] = []; + + for (let index = 0; index < document.childCount; index += 1) { + const blockId = blockIds[index]; + if (!blockId) throw new Error("Eval standup fixture has an unexpected block count."); + const node = withTiptapNodeAiRef(document.child(index), blockId); + const snapshot = await createDocumentAiBlockSnapshot(node); + blocks.set(snapshot.editRef, snapshot.content); + content.push(await serializeTiptapNodeToAiHtml(node)); + } + + return { blocks, content: content.join("") }; +} + +// Per-tool stubbed outputs. Reads return the realistic item for the requested +// path; everything else returns a neutral success so a follow-up step can still +// proceed. No real Durable Object is touched and no workspace is mutated. +async function evalToolFixture(toolName: string, input: unknown): Promise { + if (toolName === "workspace_read_items") { + const fixture = await getEvalStandupFixture(); + const requests = ( + input as { + requests?: Array<{ editRef?: string; mode?: string; path?: string }>; + } + )?.requests; + const results = (requests ?? []).map((request) => { + if (request.path !== STANDUP_PATH) { + return { code: "path_not_found", path: request.path ?? "", status: "failed" }; + } + if (request.mode === "block") { + const editRef = request.editRef ?? ""; + const content = fixture.blocks.get(editRef); + if (!content) { + return { code: "edit_ref_not_found", path: STANDUP_PATH, status: "failed" }; + } + + return { + content, + editRef, + format: "html", + itemId: "standup-document", + path: STANDUP_PATH, + status: "ready", + type: "block", + }; + } + if (request.mode !== "start") { + return { code: "invalid_selection", path: STANDUP_PATH, status: "failed" }; + } + + return { + content: fixture.content, + format: "html", + itemId: "standup-document", + location: { endBlock: 2, kind: "blocks", startBlock: 1, totalBlocks: 2 }, + path: STANDUP_PATH, + status: "ready", + type: "document", + }; + }); + + return { references: [], results }; + } + return { ok: true, note: "eval stub — no real mutation" }; } // Real workspace tools with stubbed execution. Evals grade tool *selection* and @@ -76,48 +142,56 @@ function evalToolFixture(toolName: string): unknown { // the provider-compatible schema (maxItems stripped, which Anthropic requires) via // the shared `createProviderCompatibleInputSchema`, plus the `inputExamples` the // gateway middleware injects. Only execution is stubbed. -function buildEvalToolSet(): ToolSet { +function buildEvalToolSet(canMutate: boolean): ToolSet { return Object.fromEntries( - workspaceToolDefinitions.map((definition) => [ - definition.name, - tool({ - description: definition.description, - inputSchema: createProviderCompatibleInputSchema( - asSchema(definition.inputSchema as z.ZodTypeAny), - ), - inputExamples: definition.inputExamples, - execute: async () => evalToolFixture(definition.name), - }), - ]), + workspaceToolDefinitions + .filter((definition) => canMutate || definition.access === "read") + .map((definition) => [ + definition.name, + tool({ + description: definition.description, + inputSchema: createProviderCompatibleInputSchema( + asSchema(definition.inputSchema as z.ZodTypeAny), + ), + inputExamples: definition.inputExamples, + execute: async (input: unknown) => evalToolFixture(definition.name, input), + }), + ]), ) as ToolSet; } -const EVAL_TOOL_SET = buildEvalToolSet(); - /** * Run one workspace-agent turn against a real model and return a normalized, * gradeable result. Invalid tool calls are captured (the AI SDK surfaces them as * content parts rather than throwing), then re-validated against the real schema. */ export async function runWorkspaceAgent(input: WorkspaceAgentInput): Promise { + const canMutate = input.canMutate ?? true; const modelId = resolveWorkspaceAiChatModelId( input.modelId ?? DEFAULT_WORKSPACE_AI_CHAT_MODEL_ID, ); - const system = input.system - ? `${getAIThreadSoulPrompt()}\n\n${input.system}` - : getAIThreadSoulPrompt(); + // Production-identical system text: the soul prompt, the workspace citation + // rules, and the runtime scope block that `beforeTurn` injects. Grading a + // model against a thinner prompt than production ships would measure the + // harness, not the product. + const workspacePrompt = getAIThreadSystemPromptForWorkspace( + getAIThreadSoulPrompt(), + { canMutate, workspaceName: "Study" }, + { timeZone: "America/New_York" }, + ); const result = await generateText({ model: getWorkspaceAiLanguageModel(modelId, env, "eval"), providerOptions: getWorkspaceAiGatewayProviderOptions({ modelId }), - system, + system: workspacePrompt, prompt: input.prompt, - tools: EVAL_TOOL_SET, + tools: buildEvalToolSet(canMutate), // A couple of steps so read→write flows can happen; kept small and cheap. stopWhen: stepCountIs(3), }); const toolCalls: WorkspaceAgentToolCall[] = []; + const priorReadEditRefsByPath = new Map>(); for (const step of result.steps) { for (const part of step.content) { if (part.type !== "tool-call") continue; @@ -125,8 +199,10 @@ export async function runWorkspaceAgent(input: WorkspaceAgentInput): Promise [path, [...refs]]), + ), valid: parsed ? parsed.success : false, issues: parsed ? parsed.success @@ -137,7 +213,49 @@ export async function runWorkspaceAgent(input: WorkspaceAgentInput): Promise>) { + const results = (output as { results?: unknown[] })?.results; + if (!Array.isArray(results)) { + return; } - return { text: result.text, finishReason: result.finishReason, toolCalls }; + for (const result of results) { + if (!result || typeof result !== "object") { + continue; + } + + const { content, editRef, path } = result as { + content?: unknown; + editRef?: unknown; + path?: unknown; + }; + if (typeof path !== "string") { + continue; + } + let refs = refsByPath.get(path); + if (!refs) { + refs = new Set(); + refsByPath.set(path, refs); + } + if (typeof editRef === "string") { + refs.add(editRef); + } + if (typeof content === "string") { + for (const match of content.matchAll(/data-edit-ref="([^"]+)"/g)) { + if (match[1]) { + refs.add(match[1]); + } + } + } + } } diff --git a/eval/support/scorers.ts b/eval/support/scorers.ts index 17af85dd7..8a045a8d8 100644 --- a/eval/support/scorers.ts +++ b/eval/support/scorers.ts @@ -66,36 +66,47 @@ export function scoreNoForbiddenTools( /** * For a read→edit turn: the model must submit a *targeted* edit (replace / insert / - * delete) whose `ref` came from the read fixture — not a fabricated ref, and not a - * whole-document `replace_all`. Without this, a hollow read stub plus the permissive - * `ref` schema (any nonempty string) let a hallucinated target score a false pass. + * delete) whose `editRef` came from a completed earlier read, not a fabricated + * target or a whole-document `overwrite`. */ -export function scoreTargetedEditProvenance( - output: WorkspaceAgentOutput, - validRefs: string[], -): ScoreResult { - const refs = new Set(validRefs); - const edits = output.toolCalls - .filter((call) => call.name === "workspace_edit_item") - .flatMap((call) => { - const input = call.input as { edits?: Array<{ op?: string; ref?: string }> }; - return Array.isArray(input.edits) ? input.edits : []; - }); - const isValidRef = (edit: { ref?: string }) => typeof edit.ref === "string" && refs.has(edit.ref); - const targeted = edits.filter((edit) => edit.op !== "replace_all" && isValidRef(edit)); - const fabricated = edits.filter((edit) => edit.op !== "replace_all" && !isValidRef(edit)); - const usedReplaceAll = edits.some((edit) => edit.op === "replace_all"); - const pass = targeted.length > 0 && fabricated.length === 0 && !usedReplaceAll; +export function scoreTargetedEditProvenance(output: WorkspaceAgentOutput): ScoreResult { + let targeted = 0; + const fabricated: string[] = []; + let usedOverwrite = false; + + for (const call of output.toolCalls) { + if (call.name !== "workspace_edit_item") continue; + const input = call.input as { + edits?: Array<{ editRef?: string; op?: string }>; + path?: string; + }; + if (!Array.isArray(input.edits)) continue; + + const priorReadEditRefs = new Set( + typeof input.path === "string" && Object.hasOwn(call.priorReadEditRefsByPath, input.path) + ? call.priorReadEditRefsByPath[input.path] + : [], + ); + for (const edit of input.edits) { + if (edit.op === "overwrite") { + usedOverwrite = true; + } else if (typeof edit.editRef === "string" && priorReadEditRefs.has(edit.editRef)) { + targeted += 1; + } else { + fabricated.push(edit.editRef ?? ""); + } + } + } + const pass = targeted > 0 && fabricated.length === 0 && !usedOverwrite; const reasons: string[] = []; - if (targeted.length === 0) reasons.push("no targeted edit used a ref from the read"); - if (fabricated.length > 0) - reasons.push(`fabricated ref(s): ${fabricated.map((e) => e.ref ?? "").join(", ")}`); - if (usedReplaceAll) reasons.push("used replace_all instead of a targeted edit"); + if (targeted === 0) reasons.push("no targeted edit used an editRef from the read"); + if (fabricated.length > 0) reasons.push(`fabricated editRef(s): ${fabricated.join(", ")}`); + if (usedOverwrite) reasons.push("used overwrite instead of a targeted edit"); return { score: pass ? 1 : 0, pass, - message: pass ? "targeted edit used a ref from the read fixture" : reasons.join("; "), + message: pass ? "targeted edit used an editRef from a prior read" : reasons.join("; "), }; } diff --git a/eval/workspace-tools.eval.ts b/eval/workspace-tools.eval.ts index cf3c7e274..c0b05593c 100644 --- a/eval/workspace-tools.eval.ts +++ b/eval/workspace-tools.eval.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { workspaceToolCases } from "./datasets/workspace-tools.cases"; -import { EVAL_READ_FIXTURE_REFS, runWorkspaceAgent } from "./support/harness"; +import { runWorkspaceAgent, type WorkspaceAgentOutput } from "./support/harness"; import { scoreAnswerQuality, scoreExpectedTools, @@ -10,6 +10,45 @@ import { scoreToolInputsValid, } from "./support/scorers"; +const STANDUP_LIST_REF = "b_standupList1.r_bullet0001"; +const STANDUP_PATH = "/Notes/Standup.md"; + +describe("workspace eval scorers", () => { + it.each([ + { editPath: STANDUP_PATH, expected: false, readPath: STANDUP_PATH, refs: [] }, + { + editPath: STANDUP_PATH, + expected: true, + readPath: STANDUP_PATH, + refs: [STANDUP_LIST_REF], + }, + { + editPath: "/Notes/Other.md", + expected: false, + readPath: STANDUP_PATH, + refs: [STANDUP_LIST_REF], + }, + ])("requires an editRef read from the edited path", ({ editPath, expected, readPath, refs }) => { + const output: WorkspaceAgentOutput = { + text: "", + toolCalls: [ + { + input: { + edits: [{ editRef: STANDUP_LIST_REF, op: "insert_after" }], + path: editPath, + }, + issues: [], + name: "workspace_edit_item", + priorReadEditRefsByPath: { [readPath]: refs }, + valid: true, + }, + ], + }; + + expect(scoreTargetedEditProvenance(output).pass).toBe(expected); + }); +}); + // Live evals: real model turns through the real gateway wiring. On-demand and // billed, so they live outside `pnpm test` — run with `pnpm eval`. Skip cleanly // when the gateway key is absent instead of failing. @@ -33,9 +72,9 @@ describe.skipIf(!process.env.AI_GATEWAY_API_KEY)("workspace tools", () => { } // 2b. Deterministic — a read→edit turn made a targeted edit whose ref came - // from the read fixture, not a fabricated ref or a whole-doc replace_all. + // from the read fixture, not a fabricated ref or a whole-doc overwrite. if (testCase.requiresTargetedEditFromRead) { - const provenance = scoreTargetedEditProvenance(output, EVAL_READ_FIXTURE_REFS); + const provenance = scoreTargetedEditProvenance(output); expect(provenance.pass, `edit provenance — ${provenance.message}`).toBe(true); } diff --git a/package.json b/package.json index 1ecba0ab9..9fbd0554d 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "doctor": "react-doctor . --scope changed --verbose", "doctor:full": "react-doctor . --scope full --verbose", "knip": "knip", - "prepare": "vp config --no-agent", + "prepare": "vp config --no-agent && node scripts/copy-widget-libs.mjs", "test": "vp test --run", "test:workers": "vp test --run --project=workers", "eval": "infisical run --env=dev --path=/app -- vitest --config vitest.evals.config.ts --run", @@ -90,6 +90,7 @@ "@tiptap/extension-code-block": "^3.29.2", "@tiptap/extension-collaboration": "^3.29.2", "@tiptap/extension-collaboration-caret": "^3.29.2", + "@tiptap/extension-details": "^3.29.2", "@tiptap/extension-highlight": "^3.29.2", "@tiptap/extension-horizontal-rule": "^3.29.2", "@tiptap/extension-link": "^3.29.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 04a41b2e5..67bd2a7ba 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -170,6 +170,9 @@ importers: '@tiptap/extension-collaboration-caret': specifier: ^3.29.2 version: 3.29.2(@tiptap/core@3.29.2(@tiptap/pm@3.29.2))(@tiptap/pm@3.29.2)(@tiptap/y-tiptap@3.0.8(prosemirror-model@1.25.11)(prosemirror-state@1.4.4)(prosemirror-view@1.42.2)(y-protocols@1.0.7(yjs@13.6.31))(yjs@13.6.31)) + '@tiptap/extension-details': + specifier: ^3.29.2 + version: 3.29.2(@tiptap/core@3.29.2(@tiptap/pm@3.29.2))(@tiptap/extension-text-style@3.29.2(@tiptap/core@3.29.2(@tiptap/pm@3.29.2)))(@tiptap/pm@3.29.2) '@tiptap/extension-highlight': specifier: ^3.29.2 version: 3.29.2(@tiptap/core@3.29.2(@tiptap/pm@3.29.2)) @@ -347,7 +350,7 @@ importers: version: 1.50.0(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(wrangler@4.118.0(@cloudflare/workers-types@5.20260801.1)) '@cloudflare/vitest-pool-workers': specifier: ^0.18.8 - version: 0.18.8(@cloudflare/workers-types@5.20260801.1)(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10) + version: 0.18.8(@cloudflare/workers-types@5.20260801.1)(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3))) '@cloudflare/workers-types': specifier: ^5.20260801.1 version: 5.20260801.1 @@ -4332,6 +4335,13 @@ packages: '@tiptap/y-tiptap': ^3.0.7 yjs: ^13 + '@tiptap/extension-details@3.29.2': + resolution: {integrity: sha512-W614IjlpwkAxIUp5I2ILIWWJu//tPsJuaDp52Xjso+EYF3iyH2EklOdFYebjlt1yKllXl1OVs7RCghgRm1bauQ==} + peerDependencies: + '@tiptap/core': 3.29.2 + '@tiptap/extension-text-style': 3.29.2 + '@tiptap/pm': 3.29.2 + '@tiptap/extension-document@3.29.2': resolution: {integrity: sha512-YUamvefLnsqu6124GavVTI7nqcFlQJ12ROB0oSwG69eSBZYNjg1tIs05LFrBxkwf4Xgqd6YzfJ9+FeG428RvzQ==} peerDependencies: @@ -4440,6 +4450,11 @@ packages: peerDependencies: '@tiptap/core': 3.29.2 + '@tiptap/extension-text-style@3.29.2': + resolution: {integrity: sha512-aQad9V9ROaEi3hE4g5z9wQ6lv5FSnzlnWFMVJxRjNlwXgm7H0fymxb4rGfca+pAwYkiRwpKYh1LatpJw2ECQAA==} + peerDependencies: + '@tiptap/core': 3.29.2 + '@tiptap/extension-text@3.29.2': resolution: {integrity: sha512-Ubko45JWWHe8glBt2PiGNF8hcbys/JNalFhiR7Y1X4iOOtAxAKJJxh3+eq+//NTlGuBPdWGp7zw8EEUp7anjKA==} peerDependencies: @@ -10262,7 +10277,7 @@ snapshots: - bufferutil - utf-8-validate - '@cloudflare/vitest-pool-workers@0.18.8(@cloudflare/workers-types@5.20260801.1)(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10)': + '@cloudflare/vitest-pool-workers@0.18.8(@cloudflare/workers-types@5.20260801.1)(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@26.1.2)(@vitest/browser-preview@4.1.10)(@voidzero-dev/vite-plus-core@0.2.2(@types/node@26.1.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(typescript@6.0.3)(yaml@2.9.0))(jsdom@28.1.0(@noble/hashes@2.2.0))(msw@2.14.6(@types/node@26.1.2)(typescript@6.0.3)))': dependencies: '@vitest/runner': 4.1.10 '@vitest/snapshot': 4.1.10 @@ -12990,6 +13005,12 @@ snapshots: '@tiptap/y-tiptap': 3.0.8(prosemirror-model@1.25.11)(prosemirror-state@1.4.4)(prosemirror-view@1.42.2)(y-protocols@1.0.7(yjs@13.6.31))(yjs@13.6.31) yjs: 13.6.31 + '@tiptap/extension-details@3.29.2(@tiptap/core@3.29.2(@tiptap/pm@3.29.2))(@tiptap/extension-text-style@3.29.2(@tiptap/core@3.29.2(@tiptap/pm@3.29.2)))(@tiptap/pm@3.29.2)': + dependencies: + '@tiptap/core': 3.29.2(@tiptap/pm@3.29.2) + '@tiptap/extension-text-style': 3.29.2(@tiptap/core@3.29.2(@tiptap/pm@3.29.2)) + '@tiptap/pm': 3.29.2 + '@tiptap/extension-document@3.29.2(@tiptap/core@3.29.2(@tiptap/pm@3.29.2))': dependencies: '@tiptap/core': 3.29.2(@tiptap/pm@3.29.2) @@ -13080,6 +13101,10 @@ snapshots: dependencies: '@tiptap/core': 3.29.2(@tiptap/pm@3.29.2) + '@tiptap/extension-text-style@3.29.2(@tiptap/core@3.29.2(@tiptap/pm@3.29.2))': + dependencies: + '@tiptap/core': 3.29.2(@tiptap/pm@3.29.2) + '@tiptap/extension-text@3.29.2(@tiptap/core@3.29.2(@tiptap/pm@3.29.2))': dependencies: '@tiptap/core': 3.29.2(@tiptap/pm@3.29.2) diff --git a/scripts/copy-widget-libs.mjs b/scripts/copy-widget-libs.mjs new file mode 100644 index 000000000..14a494406 --- /dev/null +++ b/scripts/copy-widget-libs.mjs @@ -0,0 +1,42 @@ +/* + * Copies KaTeX's browser build into public/ so sandboxed widget iframes can + * load it. + * + * Widgets run in an opaque-origin iframe under a strict CSP with no network + * access, so they cannot pull anything from a CDN — the assets have to be + * served from this app's own origin. KaTeX's stylesheet references its fonts + * relatively (`url(fonts/...)`), so the whole directory is copied verbatim + * rather than passed through the bundler. + * + * Runs from `prepare`, which keeps the copy in lockstep with the installed + * katex version. The output is generated, not committed. + */ +import { cp, mkdir, rm } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const require = createRequire(import.meta.url); +const katexDist = dirname(require.resolve("katex/dist/katex.min.js")); +const outDir = join( + dirname(fileURLToPath(import.meta.url)), + "..", + "public", + "widget-libs", + "katex", +); + +await rm(outDir, { recursive: true, force: true }); +await mkdir(outDir, { recursive: true }); + +for (const entry of [ + "katex.min.css", + "katex.min.js", + "fonts", + // Chemistry (\ce) and units (\pu), matching what chat and documents import. + "contrib/mhchem.min.js", +]) { + await cp(join(katexDist, entry), join(outDir, entry), { recursive: true }); +} + +console.log(`Copied KaTeX browser build to ${outDir}`); diff --git a/src/features/workspaces/ai/ai-thread-soul-prompt.ts b/src/features/workspaces/ai/ai-thread-soul-prompt.ts index 61c01bb44..1235f5909 100644 --- a/src/features/workspaces/ai/ai-thread-soul-prompt.ts +++ b/src/features/workspaces/ai/ai-thread-soul-prompt.ts @@ -46,11 +46,12 @@ export function getAIThreadSoulPrompt() { title: "Output Format", rules: [ "Format final answers as GitHub-flavored Markdown. Use concise headings, lists, blockquotes, links, tables, task lists, strikethrough, and fenced code blocks with language tags when they improve clarity.", - "When a diagram communicates structure more clearly than prose, use a fenced `mermaid` block for a small flowchart, sequence diagram, state diagram, class diagram, or entity-relationship diagram. Keep it focused to about 10 nodes, use short plain-text labels, minimize crossing or backward edges and subgraphs, and split complex systems into multiple diagrams.", + "When the user asks for a diagram or visual explanation, use a fenced `mermaid` block for a small flowchart, sequence diagram, state diagram, class diagram, or entity-relationship diagram. Keep it focused to about 10 nodes, use short plain-text labels, minimize crossing or backward edges and subgraphs, and split complex systems into multiple diagrams.", "Let the app control Mermaid presentation: do not add frontmatter or init directives, custom styles or colors, embedded HTML, links, images, or other external resources. Include a concise `accTitle` and `accDescr` describing the diagram.", "When writing Markdown with math, use `$...$` for inline math and `$$...$$` on separate lines for block math. Do not use `\\(...\\)` or `\\[...\\]`; our renderer only understands dollar-sign delimiters.", "Chemistry renders with `\\ce{...}` (e.g. `$\\ce{CH4 + 2 O2 -> CO2 + 2 H2O}$`) and quantities with units render with `\\pu{...}`.", - "CRITICAL: every literal dollar sign — every price, cost, salary, or currency figure — MUST be escaped as `\\$`. Write `\\$5`, not `$5`. Write `\\$1,000 − \\$200 = \\$800`, not `$1,000 − $200 = $800`. A missed escape turns the price into broken math on screen.", + "CRITICAL, in chat replies only: every literal dollar sign — every price, cost, salary, or currency figure — MUST be escaped as `\\$`. Write `\\$5`, not `$5`. Write `\\$1,000 − \\$200 = \\$800`, not `$1,000 − $200 = $800`. A missed escape turns the price into broken math on screen.", + "That escape is a Markdown rule and applies to chat replies only. Document and widget content is HTML, where `\\$` renders as a visible backslash — write money plainly there, as `$30`.", ], }, { diff --git a/src/features/workspaces/ai/ai-thread.ts b/src/features/workspaces/ai/ai-thread.ts index c61e451a1..e4da44214 100644 --- a/src/features/workspaces/ai/ai-thread.ts +++ b/src/features/workspaces/ai/ai-thread.ts @@ -16,6 +16,7 @@ import type { TurnContext, } from "@cloudflare/think"; import { defaultContextOverflowClassifier, Think } from "@cloudflare/think"; +import bundledSkills from "agents:skills"; import { callable } from "agents"; import { generateText, type LanguageModel, type ToolSet } from "ai"; @@ -134,6 +135,14 @@ export function createAIThreadClass(getUserAIStore: () => typeof UserAIStore) { return getAIThreadSoulPrompt(); } + // On-demand instruction bundles (progressive disclosure). The model sees + // only each skill's name/description until a task matches, then calls + // activate_skill to load the full guide. Bundled from ./skills via the + // agents Vite plugin (agents:skills virtual module). + getSkills() { + return [bundledSkills]; + } + configureSession(session: Session) { return session .withContext("soul", { diff --git a/src/features/workspaces/ai/skills/widget-authoring/SKILL.md b/src/features/workspaces/ai/skills/widget-authoring/SKILL.md new file mode 100644 index 000000000..ee3a12cf6 --- /dev/null +++ b/src/features/workspaces/ai/skills/widget-authoring/SKILL.md @@ -0,0 +1,41 @@ +--- +name: widget-authoring +description: Author or edit ThinkEx widgets, which are self-contained interactive HTML blocks inside documents. Use when the user explicitly asks for a widget, asks for interaction or live computation, or wants a document visual that ordinary blocks cannot express. +--- + +# Author ThinkEx widgets + +A widget is a document block whose HTML-escaped text content is one interactive HTML fragment: + +```html +
<style>...</style>...
+``` + +There is no separate widget item type. A document containing only this block acts as a standalone widget. + +## Create + +Follow this contract: + +1. Supply fragment content only: HTML plus inline ` +
+ +``` + +Read theme colors inside `draw()` with `getComputedStyle(document.documentElement)` when pixels must use app tokens. Call `draw()` from every handler that changes the visualization's state. diff --git a/src/features/workspaces/ai/skills/widget-authoring/references/starter.md b/src/features/workspaces/ai/skills/widget-authoring/references/starter.md new file mode 100644 index 000000000..57b348600 --- /dev/null +++ b/src/features/workspaces/ai/skills/widget-authoring/references/starter.md @@ -0,0 +1,47 @@ +# Widget starter + +Use this as the structural baseline for a new widget. Adapt the controls and behavior to the request. Keep the content root unframed because ThinkEx supplies the outer title, border, and gutter. + +```html + +
+
Ready.
+
+ +
+
+ +``` + +Before writing the block, replace placeholder behavior and labels, wire every visible control, and HTML-escape the complete fragment inside the widget element. diff --git a/src/features/workspaces/ai/workspace-citations.test.ts b/src/features/workspaces/ai/workspace-citations.test.ts index 38cc76804..4abc04851 100644 --- a/src/features/workspaces/ai/workspace-citations.test.ts +++ b/src/features/workspaces/ai/workspace-citations.test.ts @@ -68,7 +68,7 @@ describe("workspace citations", () => { references: [first], results: [ { - content: '

Notes

', + content: '

Notes

', format: "html", itemId: "item-1", location: { diff --git a/src/features/workspaces/components/WorkspaceContent.tsx b/src/features/workspaces/components/WorkspaceContent.tsx index b85bb6468..fb4a8b59e 100644 --- a/src/features/workspaces/components/WorkspaceContent.tsx +++ b/src/features/workspaces/components/WorkspaceContent.tsx @@ -1,5 +1,5 @@ import { Eye, FolderOpen } from "lucide-react"; -import { useRef, useState } from "react"; +import { useMemo, useRef, useState } from "react"; import { ContextMenu, ContextMenuContent, ContextMenuTrigger } from "#/components/ui/context-menu"; import { @@ -40,7 +40,11 @@ import { useWorkspaceMutationAccess } from "#/features/workspaces/components/wor import { useWorkspaceViewCapabilities } from "#/features/workspaces/components/workspace-view-policy"; import type { WorkspaceItemType, WorkspaceSummary } from "#/features/workspaces/contracts"; import { getWorkspaceItemDisplay } from "#/features/workspaces/model/item-display"; -import { getWorkspaceChildren, splitWorkspaceChildren } from "#/features/workspaces/model/tree"; +import { + getWorkspaceChildren, + getWorkspaceItemPath, + splitWorkspaceChildren, +} from "#/features/workspaces/model/tree"; import type { WorkspaceItem } from "#/features/workspaces/model/types"; import { getWorkspaceBrowseParentId, isWorkspaceItemView } from "#/features/workspaces/model/view"; import { workspaceUploadTypeLabel } from "#/features/workspaces/upload/workspace-upload-intake"; @@ -69,11 +73,13 @@ export default function WorkspaceContent({ }: WorkspaceContentProps) { const workspaceId = workspace.id; const actionDialogs = useWorkspaceItemActionDialogState(); + const itemsById = useMemo(() => new Map(items.map((item) => [item.id, item])), [items]); if (isWorkspaceItemView(activeItem)) { return ( <> { const existing = current[slotId]; if ( existing?.kind === "document" && existing.canEdit === canEdit && + existing.documentPath === documentPath && existing.editor === editor && existing.itemId === itemId && - existing.slotId === slotId + existing.slotId === slotId && + existing.workspaceId === workspaceId ) { return current; } @@ -101,7 +117,7 @@ export function useDocumentEditorToolbar({ return next; }); }; - }, [canEdit, editor, itemId, slotId, setRegistration]); + }, [canEdit, documentPath, editor, itemId, slotId, workspaceId, setRegistration]); } export function useFileItemToolbar({ @@ -194,8 +210,10 @@ export function WorkspaceItemToolbarSlot({ {registration.kind === "document" ? ( ) : ( (null); const input = useWorkspaceAiComposerDraftText(activeThreadId); const setDraftText = useWorkspaceAiComposerDraftStore((state) => state.setText); - const setInput = (value: SetStateAction) => setDraftText(activeThreadId, value); + const setInput = useCallback( + (value: SetStateAction) => setDraftText(activeThreadId, value), + [activeThreadId, setDraftText], + ); + const focusRequest = useWorkspaceAiComposerFocusRequest(activeThreadId); + const clearFocusRequest = useWorkspaceAiComposerDraftStore((state) => state.clearFocusRequest); const dictation = useAiChatDictation({ input, setInput }); const draftFiles = useWorkspaceAiComposerDraftFiles(activeThreadId); const attachmentsReady = @@ -130,6 +144,25 @@ export default function AiChatPromptInput({ setInput, textareaRef, }); + useEffect(() => { + if (focusRequest === 0) { + return; + } + + const frame = requestAnimationFrame(() => { + const textarea = textareaRef.current; + if (!textarea) { + return; + } + + textarea.focus(); + const caret = textarea.value.length; + textarea.setSelectionRange(caret, caret); + clearFocusRequest(activeThreadId, focusRequest); + }); + + return () => cancelAnimationFrame(frame); + }, [activeThreadId, clearFocusRequest, focusRequest]); const attachments: Omit = { add: addFiles, diff --git a/src/features/workspaces/components/document-editor/DocumentEditorSurface.tsx b/src/features/workspaces/components/document-editor/DocumentEditorSurface.tsx index 017fe3db4..01b59373e 100644 --- a/src/features/workspaces/components/document-editor/DocumentEditorSurface.tsx +++ b/src/features/workspaces/components/document-editor/DocumentEditorSurface.tsx @@ -5,12 +5,14 @@ import { useQuery } from "@tanstack/react-query"; import { useState } from "react"; import { Skeleton } from "#/components/ui/skeleton"; +import { stageComposerPrompt } from "#/features/workspaces/composer/workspace-composer-actions"; import { DocumentAskSelectionMenu } from "#/features/workspaces/components/document-editor/DocumentAskSelectionMenu"; import { DocumentWordCount } from "#/features/workspaces/components/document-editor/DocumentWordCount"; import { useDocumentEditorToolbar } from "#/features/workspaces/components/WorkspaceItemToolbarSlot"; import { useWorkspacePaneRuntime } from "#/features/workspaces/components/WorkspacePaneRuntime"; import { useWorkspaceMutationAccess } from "#/features/workspaces/components/workspace-mutation-access"; import { DocumentEditReviewExtension } from "#/features/workspaces/documents/document-edit-review-extension"; +import { DocumentWidgetActionProvider } from "#/features/workspaces/documents/document-widget-node"; import { getTiptapDocumentBaseExtensions, tiptapDocumentYjsField, @@ -25,10 +27,12 @@ import { DEFAULT_COLLABORATION_COLOR } from "#/lib/design-system-colors"; import { getAuthSessionQueryOptions } from "#/lib/session-query"; export function DocumentEditorSurface({ + documentPath, item, viewInstanceId, workspaceId, }: { + documentPath: string; item: WorkspaceItem; viewInstanceId: string; workspaceId: string; @@ -50,6 +54,7 @@ export function DocumentEditorSurface({ return ( { if (event.key !== "Escape" || !paneRuntime?.onCloseItemView) { @@ -102,9 +109,11 @@ function DocumentEditorInstance({ useDocumentEditorToolbar({ canEdit: capabilities.canMutateContent, + documentPath, editor: capabilities.canMutateContent ? editor : null, itemId: item.id, slotId: viewInstanceId, + workspaceId, }); useDocumentEditReviewOverlay({ canEdit: capabilities.canMutateContent, @@ -122,7 +131,19 @@ function DocumentEditorInstance({ scrollTarget={scrollTarget} workspaceId={workspaceId} /> - + + stageComposerPrompt( + workspaceId, + `A widget in ${documentPath} hit this error. Please fix it:\n\n${error}`, + ) + : undefined + } + > + + diff --git a/src/features/workspaces/components/document-editor/DocumentToolbar.tsx b/src/features/workspaces/components/document-editor/DocumentToolbar.tsx index 4dbbf8a1c..d2c90c0fe 100644 --- a/src/features/workspaces/components/document-editor/DocumentToolbar.tsx +++ b/src/features/workspaces/components/document-editor/DocumentToolbar.tsx @@ -1,6 +1,6 @@ import type { Editor } from "@tiptap/react"; -import { Check, Download, EllipsisVertical, FileText, Redo2, Undo2 } from "lucide-react"; -import type { ReactNode } from "react"; +import { Check, Download, EllipsisVertical, FileText, Redo2, Shapes, Undo2 } from "lucide-react"; +import { type ReactNode, useState } from "react"; import { Button } from "#/components/ui/button"; import { DocumentEditUndoButton } from "#/features/workspaces/components/document-editor/DocumentEditUndoButton"; @@ -32,6 +32,7 @@ import { getTextAlignIcon, isCodeBlock, } from "#/features/workspaces/components/document-editor/document-editor-toolbar-actions"; +import { WorkspaceAddWidgetDialog } from "#/features/workspaces/components/widget/WorkspaceAddWidgetDialog"; import { WorkspaceResponsiveToolbar, WorkspaceToolbarIconButton, @@ -40,14 +41,19 @@ import { workspaceToolbarTextButtonSizeClass } from "#/features/workspaces/compo export function DocumentToolbar({ canEdit, + documentPath, editor, itemId, + workspaceId, }: { canEdit: boolean; + documentPath: string; editor: Editor | null; itemId: string; + workspaceId: string; }) { const editorState = useDocumentEditorUiState(editor); + const [addWidgetOpen, setAddWidgetOpen] = useState(false); const { activeReview, hideReview } = useDocumentEditReview(); // Reviewing borrows the toolbar rather than floating over the page: the @@ -77,31 +83,48 @@ export function DocumentToolbar({ } return ( - } - mobileContentClassName="max-h-[min(var(--available-height),28rem)] w-64 max-w-[calc(100dvw-2rem)] overscroll-contain" - scrollable - > - - - - editor?.chain().focus().undo().run()} - > - - - editor?.chain().focus().redo().run()} + <> + setAddWidgetOpen(true)} + /> + } + mobileContentClassName="max-h-[min(var(--available-height),28rem)] w-64 max-w-[calc(100dvw-2rem)] overscroll-contain" + scrollable > - - - - + + + + editor?.chain().focus().undo().run()} + > + + + editor?.chain().focus().redo().run()} + > + + + setAddWidgetOpen(true)}> + + + + + + ); } @@ -149,9 +172,11 @@ function DocumentEditReviewControls({ function DocumentMobileMenuContent({ editor, editorState, + onAddWidget, }: { editor: Editor | null; editorState: DocumentEditorUiState; + onAddWidget: () => void; }) { return ( <> @@ -195,6 +220,7 @@ function DocumentMobileMenuContent({ label="Redo" onClick={() => editor?.chain().focus().redo().run()} /> + } label="Add widget" onClick={onAddWidget} /> diff --git a/src/features/workspaces/components/widget/WorkspaceAddWidgetDialog.tsx b/src/features/workspaces/components/widget/WorkspaceAddWidgetDialog.tsx new file mode 100644 index 000000000..203187018 --- /dev/null +++ b/src/features/workspaces/components/widget/WorkspaceAddWidgetDialog.tsx @@ -0,0 +1,92 @@ +import { useId } from "react"; + +import { Button } from "#/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "#/components/ui/dialog"; +import { Field, FieldGroup, FieldLabel } from "#/components/ui/field"; +import { Textarea } from "#/components/ui/textarea"; +import { stageComposerPrompt } from "#/features/workspaces/composer/workspace-composer-actions"; + +/** + * Adding a widget is an AI-authoring kickoff, not a blank block. Rather than + * insert an empty widget the user would have to fill in by hand, we collect a + * description and hand it to the AI by prefilling the composer (via the shared + * `stageComposerPrompt` primitive). The AI writes the widget into the document. + * + * The prompt names the document by path because it is staged, not sent: the + * user may switch views before sending, which would leave "this document" + * pointing somewhere else. + */ +export function WorkspaceAddWidgetDialog({ + documentPath, + open, + workspaceId, + onOpenChange, +}: { + documentPath: string; + open: boolean; + workspaceId: string; + onOpenChange: (open: boolean) => void; +}) { + const descriptionId = useId(); + + return ( + + {open ? ( + +
{ + const raw = formData.get("description"); + const description = (typeof raw === "string" ? raw : "").trim(); + + if (!description) { + return; + } + + stageComposerPrompt( + workspaceId, + `Add an interactive widget to ${documentPath}: ${description}`, + ); + onOpenChange(false); + }} + > + + Add a widget + + A widget is an interactive tool that lives in this document, such as a simulation, + calculator, diagram, or visualization. Describe what you want, and the AI will build + it for you. + + + + + What should it do? +