From 5938ab64d652857932a5b6b1dabcc361014939e6 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 4 Jul 2026 20:25:19 +0000 Subject: [PATCH 1/2] feat(ai-core): add provider abstraction for Session Notes + Clips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New @pairux/ai-core package — the compute-plane contract for PairUX Pro AI Session Notes and AI Clips (PRD "Replay"). - AiProvider interface: summarizeSession() + selectClips(), text in / validated out - Two drivers: anthropic (managed key or BYOK) and ollama (fully local) - Zod schemas (SessionNote, ClipCandidate[]) as the single source of truth - One-shot auto-repair on malformed replies, then StructuredOutputError so the caller can fall back to manual mode (the PRD's clip-selection contract) - Only transcript text ever crosses a provider boundary; no SDK dependency — the desktop host injects a structurally-typed Anthropic client - Managed default model is Haiku 4.5 to keep the Pro unit economics in envelope (review finding F3) Standalone package — does not touch existing apps. Verified in isolation: typecheck, eslint (0 warnings), build, and 37 unit tests all pass. Co-Authored-By: Claude Opus 4.8 --- packages/ai-core/README.md | 50 ++++++++ packages/ai-core/package.json | 32 +++++ packages/ai-core/src/complete.test.ts | 40 +++++++ packages/ai-core/src/complete.ts | 34 ++++++ packages/ai-core/src/config.ts | 14 +++ packages/ai-core/src/errors.ts | 15 +++ packages/ai-core/src/index.ts | 20 ++++ packages/ai-core/src/parse.test.ts | 52 ++++++++ packages/ai-core/src/parse.ts | 52 ++++++++ packages/ai-core/src/prompts.test.ts | 41 +++++++ packages/ai-core/src/prompts.ts | 36 ++++++ packages/ai-core/src/providers/anthropic.ts | 69 +++++++++++ packages/ai-core/src/providers/factory.ts | 18 +++ packages/ai-core/src/providers/ollama.ts | 70 +++++++++++ .../ai-core/src/providers/providers.test.ts | 111 ++++++++++++++++++ packages/ai-core/src/schemas.test.ts | 70 +++++++++++ packages/ai-core/src/schemas.ts | 51 ++++++++ packages/ai-core/src/templates.ts | 35 ++++++ packages/ai-core/src/transcript.test.ts | 30 +++++ packages/ai-core/src/transcript.ts | 19 +++ packages/ai-core/src/types.ts | 49 ++++++++ packages/ai-core/tsconfig.json | 12 ++ packages/ai-core/vitest.config.ts | 9 ++ pnpm-lock.yaml | 47 +++++--- 24 files changed, 957 insertions(+), 19 deletions(-) create mode 100644 packages/ai-core/README.md create mode 100644 packages/ai-core/package.json create mode 100644 packages/ai-core/src/complete.test.ts create mode 100644 packages/ai-core/src/complete.ts create mode 100644 packages/ai-core/src/config.ts create mode 100644 packages/ai-core/src/errors.ts create mode 100644 packages/ai-core/src/index.ts create mode 100644 packages/ai-core/src/parse.test.ts create mode 100644 packages/ai-core/src/parse.ts create mode 100644 packages/ai-core/src/prompts.test.ts create mode 100644 packages/ai-core/src/prompts.ts create mode 100644 packages/ai-core/src/providers/anthropic.ts create mode 100644 packages/ai-core/src/providers/factory.ts create mode 100644 packages/ai-core/src/providers/ollama.ts create mode 100644 packages/ai-core/src/providers/providers.test.ts create mode 100644 packages/ai-core/src/schemas.test.ts create mode 100644 packages/ai-core/src/schemas.ts create mode 100644 packages/ai-core/src/templates.ts create mode 100644 packages/ai-core/src/transcript.test.ts create mode 100644 packages/ai-core/src/transcript.ts create mode 100644 packages/ai-core/src/types.ts create mode 100644 packages/ai-core/tsconfig.json create mode 100644 packages/ai-core/vitest.config.ts diff --git a/packages/ai-core/README.md b/packages/ai-core/README.md new file mode 100644 index 00000000..acb9496a --- /dev/null +++ b/packages/ai-core/README.md @@ -0,0 +1,50 @@ +# @pairux/ai-core + +Provider abstraction for PairUX Pro **AI Session Notes** and **AI Clips**. This is the +compute-plane contract the desktop host calls into: transcript text in, validated +structured output out. + +## What it does + +- **One interface** — `AiProvider` with `summarizeSession()` and `selectClips()`. +- **Two drivers** — `anthropic` (managed key **or** BYOK) and `ollama` (fully local). +- **Validated output** — every reply is parsed against Zod schemas (`SessionNote`, + `ClipCandidate[]`). Malformed output triggers exactly one auto-repair retry, then + throws `StructuredOutputError` so the caller can fall back to manual mode — the + contract the PRD specifies for clip selection. +- **Text-only boundary** — a provider only ever receives `TranscriptInput` (text + + metadata). Audio and video never cross this package, by construction. + +## Usage + +```ts +import Anthropic from '@anthropic-ai/sdk'; +import { createProvider } from '@pairux/ai-core'; + +// Managed key or BYOK — the host injects the real SDK client. +const provider = createProvider({ kind: 'anthropic', client: new Anthropic() }); + +const note = await provider.summarizeSession(transcript, { template: 'pair-programming' }); +const clips = await provider.selectClips(transcript, { platform: 'shorts', maxClips: 5 }); +``` + +Fully-local mode (nothing leaves the device): + +```ts +const provider = createProvider({ kind: 'ollama', model: 'llama3.1' }); +``` + +## Notes + +- The managed default model is **Haiku 4.5** — see `config.ts`. This keeps the + summarize + clip-select pair (two transcript-text-only calls per session) inside + the Pro unit-economics envelope; BYOK callers can override for higher quality. +- `ai-core` deliberately does **not** depend on `@anthropic-ai/sdk`. The host passes a + client that structurally matches `AnthropicClientLike`, keeping this package light + and unit-testable offline. + +## Scope + +This is the foundation slice for the "Replay" PRD. Recording (P0-1), local Whisper +transcription (P0-2), and the ffmpeg clip renderer (P0-4) live in the desktop app and +depend on this contract; they are not part of this package. diff --git a/packages/ai-core/package.json b/packages/ai-core/package.json new file mode 100644 index 00000000..11bf13f8 --- /dev/null +++ b/packages/ai-core/package.json @@ -0,0 +1,32 @@ +{ + "name": "@pairux/ai-core", + "version": "0.0.0", + "private": true, + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "tsc", + "dev": "tsc --watch", + "typecheck": "tsc --noEmit", + "lint": "eslint src/", + "lint:fix": "eslint src/ --fix", + "test": "vitest run", + "test:watch": "vitest", + "test:coverage": "vitest run --coverage", + "clean": "rm -rf dist" + }, + "dependencies": { + "zod": "^3.24.0" + }, + "devDependencies": { + "typescript": "^5.7.0", + "vitest": "^3.2.0" + } +} diff --git a/packages/ai-core/src/complete.test.ts b/packages/ai-core/src/complete.test.ts new file mode 100644 index 00000000..b2a35c9f --- /dev/null +++ b/packages/ai-core/src/complete.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; +import { completeStructured, type CompleteFn } from './complete.js'; +import { StructuredOutputError } from './errors.js'; + +const schema = z.object({ ok: z.boolean() }); + +describe('completeStructured', () => { + it('returns parsed output when the first reply is valid', async () => { + const complete: CompleteFn = vi.fn(() => Promise.resolve('{"ok": true}')); + await expect(completeStructured(complete, { system: 's', user: 'u' }, schema)).resolves.toEqual({ ok: true }); + expect(complete).toHaveBeenCalledTimes(1); + }); + + it('retries exactly once and succeeds on the repair', async () => { + const complete = vi + .fn() + .mockResolvedValueOnce('sorry, here: not json') + .mockResolvedValueOnce('{"ok": false}'); + const result = await completeStructured(complete, { system: 's', user: 'u' }, schema); + expect(result).toEqual({ ok: false }); + expect(complete).toHaveBeenCalledTimes(2); + }); + + it('asks for JSON-only on the repair attempt', async () => { + const complete = vi.fn().mockResolvedValueOnce('nope').mockResolvedValueOnce('{"ok": true}'); + await completeStructured(complete, { system: 's', user: 'u' }, schema); + const secondCall = complete.mock.calls[1]?.[0]; + expect(secondCall?.user).toContain('ONLY the JSON'); + expect(secondCall?.system).toBe('s'); + }); + + it('rethrows StructuredOutputError after a second failure (caller falls back to manual mode)', async () => { + const complete: CompleteFn = vi.fn(() => Promise.resolve('still not json')); + await expect(completeStructured(complete, { system: 's', user: 'u' }, schema)).rejects.toBeInstanceOf( + StructuredOutputError, + ); + expect(complete).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/ai-core/src/complete.ts b/packages/ai-core/src/complete.ts new file mode 100644 index 00000000..1a962b29 --- /dev/null +++ b/packages/ai-core/src/complete.ts @@ -0,0 +1,34 @@ +import type { ZodType } from 'zod'; +import { StructuredOutputError } from './errors.js'; +import { parseStructured } from './parse.js'; + +export interface Prompt { + system: string; + user: string; +} + +/** A minimal chat completion: text in, text out. Every provider reduces to this. */ +export type CompleteFn = (prompt: Prompt) => Promise; + +/** + * Run a prompt and validate the reply against `schema`. + * + * Implements the PRD's contract: malformed output triggers exactly one + * auto-repair retry (re-asking for JSON only); a second failure rethrows the + * {@link StructuredOutputError} so the caller can fall back to manual mode. + */ +export async function completeStructured(complete: CompleteFn, prompt: Prompt, schema: ZodType): Promise { + const first = await complete(prompt); + try { + return parseStructured(first, schema); + } catch (error) { + if (!(error instanceof StructuredOutputError)) { + throw error; + } + const repaired = await complete({ + system: prompt.system, + user: `${prompt.user}\n\nYour previous reply could not be parsed. Reply again with ONLY the JSON value — no prose, no explanation, no code fences.`, + }); + return parseStructured(repaired, schema); + } +} diff --git a/packages/ai-core/src/config.ts b/packages/ai-core/src/config.ts new file mode 100644 index 00000000..46df4478 --- /dev/null +++ b/packages/ai-core/src/config.ts @@ -0,0 +1,14 @@ +/** + * Provider defaults. + * + * The managed-key default is deliberately Haiku 4.5. Summarize + clip-select is + * ~2 transcript-text-only calls per session; at Haiku pricing that keeps the + * managed path inside the Pro unit-economics envelope (review finding F3 — the + * "<$0.40/user/mo" guardrail is only reachable at Haiku tier). BYOK callers can + * override `model` with any Anthropic model for higher-quality summaries. + */ +export const DEFAULT_ANTHROPIC_MODEL = 'claude-haiku-4-5'; +export const DEFAULT_OLLAMA_MODEL = 'llama3.1'; +export const DEFAULT_OLLAMA_BASE_URL = 'http://127.0.0.1:11434'; +export const DEFAULT_MAX_TOKENS = 2048; +export const DEFAULT_MAX_CLIPS = 6; diff --git a/packages/ai-core/src/errors.ts b/packages/ai-core/src/errors.ts new file mode 100644 index 00000000..24a9f56c --- /dev/null +++ b/packages/ai-core/src/errors.ts @@ -0,0 +1,15 @@ +/** + * Thrown when a provider's response cannot be parsed into the required structured + * shape. Callers catch this to fall back to manual mode (e.g. the clip-review UI + * lets the host cut clips by hand when selection fails). + */ +export class StructuredOutputError extends Error { + /** The raw model text that failed to parse, preserved for logging and retry. */ + readonly raw: string; + + constructor(message: string, raw: string, options?: ErrorOptions) { + super(message, options); + this.name = 'StructuredOutputError'; + this.raw = raw; + } +} diff --git a/packages/ai-core/src/index.ts b/packages/ai-core/src/index.ts new file mode 100644 index 00000000..67bf6fb4 --- /dev/null +++ b/packages/ai-core/src/index.ts @@ -0,0 +1,20 @@ +/** + * @pairux/ai-core — provider abstraction for PairUX Pro AI Session Notes + Clips. + * + * One interface ({@link AiProvider}), two drivers (Anthropic for managed/BYOK, + * Ollama for fully-local), Zod-validated structured output, and a one-shot + * auto-repair on malformed replies. Only transcript text ever crosses a provider + * boundary. + */ +export * from './types.js'; +export * from './schemas.js'; +export * from './templates.js'; +export * from './config.js'; +export * from './errors.js'; +export * from './parse.js'; +export * from './transcript.js'; +export * from './complete.js'; +export * from './prompts.js'; +export * from './providers/anthropic.js'; +export * from './providers/ollama.js'; +export * from './providers/factory.js'; diff --git a/packages/ai-core/src/parse.test.ts b/packages/ai-core/src/parse.test.ts new file mode 100644 index 00000000..1ebee106 --- /dev/null +++ b/packages/ai-core/src/parse.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { StructuredOutputError } from './errors.js'; +import { extractJson, parseStructured } from './parse.js'; + +const schema = z.object({ ok: z.boolean() }); + +describe('extractJson', () => { + it('unwraps a ```json fence', () => { + expect(extractJson('```json\n{"ok": true}\n```')).toBe('{"ok": true}'); + }); + + it('unwraps a bare ``` fence', () => { + expect(extractJson('```\n[1, 2]\n```')).toBe('[1, 2]'); + }); + + it('slices JSON out of surrounding prose', () => { + expect(extractJson('Sure! {"ok": true} hope that helps')).toBe('{"ok": true}'); + }); + + it('slices an array out of prose', () => { + expect(extractJson('here you go: [{"a":1}] done')).toBe('[{"a":1}]'); + }); + + it('returns trimmed input when no JSON is present', () => { + expect(extractJson(' no json here ')).toBe('no json here'); + }); +}); + +describe('parseStructured', () => { + it('parses and validates', () => { + expect(parseStructured('{"ok": true}', schema)).toEqual({ ok: true }); + }); + + it('throws StructuredOutputError on invalid JSON', () => { + expect(() => parseStructured('{not json', schema)).toThrow(StructuredOutputError); + }); + + it('throws StructuredOutputError on schema mismatch', () => { + expect(() => parseStructured('{"ok": "yes"}', schema)).toThrow(StructuredOutputError); + }); + + it('preserves the raw output on the error', () => { + try { + parseStructured('garbage', schema); + expect.unreachable('should have thrown'); + } catch (error) { + expect(error).toBeInstanceOf(StructuredOutputError); + expect((error as StructuredOutputError).raw).toBe('garbage'); + } + }); +}); diff --git a/packages/ai-core/src/parse.ts b/packages/ai-core/src/parse.ts new file mode 100644 index 00000000..ca636b4f --- /dev/null +++ b/packages/ai-core/src/parse.ts @@ -0,0 +1,52 @@ +import type { ZodType } from 'zod'; +import { StructuredOutputError } from './errors.js'; + +/** + * Pull the first JSON value out of a model reply that may wrap it in prose or a + * ```json fence. Best-effort: returns the most plausible JSON substring, which + * `parseStructured` then validates. + */ +export function extractJson(raw: string): string { + const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(raw); + const fencedBody = fenced?.[1]; + if (fencedBody !== undefined) { + return fencedBody.trim(); + } + + const start = raw.search(/[[{]/); + if (start === -1) { + return raw.trim(); + } + + const opener = raw[start]; + if (opener === undefined) { + return raw.trim(); + } + + const closer = opener === '[' ? ']' : '}'; + const end = raw.lastIndexOf(closer); + if (end > start) { + return raw.slice(start, end + 1).trim(); + } + return raw.slice(start).trim(); +} + +/** Extract + JSON.parse + schema-validate. Throws {@link StructuredOutputError} on any failure. */ +export function parseStructured(raw: string, schema: ZodType): T { + const json = extractJson(raw); + + let parsed: unknown; + try { + parsed = JSON.parse(json) as unknown; + } catch (cause) { + throw new StructuredOutputError('model output was not valid JSON', raw, { cause }); + } + + const result = schema.safeParse(parsed); + if (!result.success) { + throw new StructuredOutputError(`model output failed schema validation: ${result.error.message}`, raw, { + cause: result.error, + }); + } + return result.data; +} diff --git a/packages/ai-core/src/prompts.test.ts b/packages/ai-core/src/prompts.test.ts new file mode 100644 index 00000000..db58f03a --- /dev/null +++ b/packages/ai-core/src/prompts.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest'; +import { buildClipPrompt, buildSummaryPrompt } from './prompts.js'; +import type { TranscriptInput } from './types.js'; + +const input: TranscriptInput = { + title: 'Debugging auth', + segments: [ + { startMs: 0, endMs: 4000, speaker: 'Host', text: 'The token check is off by one.' }, + { startMs: 4000, endMs: 9000, speaker: 'Viewer-1', text: 'Agreed, let us pin the clock.' }, + ], + sceneChangesMs: [4000], +}; + +describe('buildSummaryPrompt', () => { + it('includes the session title, transcript, and template guidance', () => { + const prompt = buildSummaryPrompt(input, { template: 'support-session' }); + expect(prompt.user).toContain('Debugging auth'); + expect(prompt.user).toContain('[00:00 Host] The token check is off by one.'); + expect(prompt.system).toContain('resolution applied'); + expect(prompt.system).toContain('"tldr"'); + }); + + it('defaults to the pair-programming template', () => { + const prompt = buildSummaryPrompt(input); + expect(prompt.system).toContain('technical decisions'); + }); +}); + +describe('buildClipPrompt', () => { + it('honors maxClips, platform bias, and scene changes', () => { + const prompt = buildClipPrompt(input, { platform: 'tiktok', maxClips: 3 }); + expect(prompt.system).toContain('select the 3 best'); + expect(prompt.system).toContain('tiktok'); + expect(prompt.user).toContain('Scene-change timestamps (ms): 4000'); + }); + + it('omits the scene-change line when there are none', () => { + const prompt = buildClipPrompt({ segments: input.segments }); + expect(prompt.user).not.toContain('Scene-change'); + }); +}); diff --git a/packages/ai-core/src/prompts.ts b/packages/ai-core/src/prompts.ts new file mode 100644 index 00000000..27032aed --- /dev/null +++ b/packages/ai-core/src/prompts.ts @@ -0,0 +1,36 @@ +import { DEFAULT_MAX_CLIPS } from './config.js'; +import type { Prompt } from './complete.js'; +import { getTemplate } from './templates.js'; +import { formatTranscript } from './transcript.js'; +import type { SelectClipsOptions, SummarizeOptions, TranscriptInput } from './types.js'; + +const SUMMARY_SHAPE = `{ + "tldr": string, + "decisions": string[], + "actionItems": [{ "description": string, "owner": string | null, "deadline": string | null, "timestampMs": number | null }], + "topics": string[] +}`; + +const CLIP_SHAPE = `[{ "startMs": number, "endMs": number, "title": string, "hookCaption": string, "platformFit": string[] }]`; + +/** Build the summarization prompt: template guidance + transcript, JSON-only reply. */ +export function buildSummaryPrompt(input: TranscriptInput, options?: SummarizeOptions): Prompt { + const template = getTemplate(options?.template); + const system = `You write structured notes from a session transcript. ${template.guidance} Reply with a single JSON object and nothing else, matching exactly this shape:\n${SUMMARY_SHAPE}`; + const header = input.title !== undefined ? `Session: ${input.title}\n\n` : ''; + const user = `${header}Transcript:\n${formatTranscript(input)}`; + return { system, user }; +} + +/** Build the clip-selection prompt: highlight instructions + optional scene changes. */ +export function buildClipPrompt(input: TranscriptInput, options?: SelectClipsOptions): Prompt { + const maxClips = options?.maxClips ?? DEFAULT_MAX_CLIPS; + const platformLine = options?.platform !== undefined ? ` Bias selection toward content that performs well on ${options.platform}.` : ''; + const sceneLine = + input.sceneChangesMs !== undefined && input.sceneChangesMs.length > 0 + ? `\n\nScene-change timestamps (ms): ${input.sceneChangesMs.join(', ')}` + : ''; + const system = `You select the ${String(maxClips)} best short highlight moments from a session transcript for social clips.${platformLine} Each highlight must be self-contained and under 60 seconds. "platformFit" values come from: shorts, tiktok, reels, x, linkedin, youtube. Reply with a single JSON array and nothing else, matching exactly this shape:\n${CLIP_SHAPE}`; + const user = `Transcript with timestamps:\n${formatTranscript(input)}${sceneLine}`; + return { system, user }; +} diff --git a/packages/ai-core/src/providers/anthropic.ts b/packages/ai-core/src/providers/anthropic.ts new file mode 100644 index 00000000..de577df2 --- /dev/null +++ b/packages/ai-core/src/providers/anthropic.ts @@ -0,0 +1,69 @@ +import { completeStructured, type CompleteFn } from '../complete.js'; +import { DEFAULT_ANTHROPIC_MODEL, DEFAULT_MAX_TOKENS } from '../config.js'; +import { buildClipPrompt, buildSummaryPrompt } from '../prompts.js'; +import { clipCandidatesSchema, sessionNoteSchema, type ClipCandidate, type SessionNote } from '../schemas.js'; +import type { AiProvider, SelectClipsOptions, SummarizeOptions, TranscriptInput } from '../types.js'; + +/** + * Structural view of the Anthropic Messages client. `ai-core` does not depend on + * `@anthropic-ai/sdk` directly — the desktop host injects a real `new Anthropic()` + * (managed key or BYOK). This keeps the package light and trivially testable, and + * makes the "only text crosses this boundary" guarantee explicit in the types. + */ +export interface AnthropicContentBlock { + readonly type: string; + readonly text?: string; +} + +export interface AnthropicMessageResponse { + readonly content: readonly AnthropicContentBlock[]; +} + +export interface AnthropicCreateBody { + model: string; + max_tokens: number; + system: string; + messages: { role: 'user'; content: string }[]; +} + +export interface AnthropicMessagesApi { + create(body: AnthropicCreateBody): Promise; +} + +export interface AnthropicClientLike { + readonly messages: AnthropicMessagesApi; +} + +export interface AnthropicProviderOptions { + client: AnthropicClientLike; + /** Defaults to Haiku 4.5 — see config.ts / review finding F3. */ + model?: string; + maxTokens?: number; +} + +export function createAnthropicProvider(options: AnthropicProviderOptions): AiProvider { + const model = options.model ?? DEFAULT_ANTHROPIC_MODEL; + const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS; + const { client } = options; + + const complete: CompleteFn = async ({ system, user }) => { + const response = await client.messages.create({ + model, + max_tokens: maxTokens, + system, + messages: [{ role: 'user', content: user }], + }); + return response.content.map((block) => (block.type === 'text' ? (block.text ?? '') : '')).join(''); + }; + + return { + kind: 'anthropic', + model, + summarizeSession(input: TranscriptInput, summarizeOptions?: SummarizeOptions): Promise { + return completeStructured(complete, buildSummaryPrompt(input, summarizeOptions), sessionNoteSchema); + }, + selectClips(input: TranscriptInput, selectOptions?: SelectClipsOptions): Promise { + return completeStructured(complete, buildClipPrompt(input, selectOptions), clipCandidatesSchema); + }, + }; +} diff --git a/packages/ai-core/src/providers/factory.ts b/packages/ai-core/src/providers/factory.ts new file mode 100644 index 00000000..6e597ac2 --- /dev/null +++ b/packages/ai-core/src/providers/factory.ts @@ -0,0 +1,18 @@ +import type { AiProvider } from '../types.js'; +import { createAnthropicProvider, type AnthropicProviderOptions } from './anthropic.js'; +import { createOllamaProvider, type OllamaProviderOptions } from './ollama.js'; + +/** Discriminated config for {@link createProvider}. */ +export type AiProviderConfig = + | ({ readonly kind: 'anthropic' } & AnthropicProviderOptions) + | ({ readonly kind: 'ollama' } & OllamaProviderOptions); + +/** Build the configured provider. `anthropic` covers both managed keys and BYOK. */ +export function createProvider(config: AiProviderConfig): AiProvider { + switch (config.kind) { + case 'anthropic': + return createAnthropicProvider(config); + case 'ollama': + return createOllamaProvider(config); + } +} diff --git a/packages/ai-core/src/providers/ollama.ts b/packages/ai-core/src/providers/ollama.ts new file mode 100644 index 00000000..ac1f63bc --- /dev/null +++ b/packages/ai-core/src/providers/ollama.ts @@ -0,0 +1,70 @@ +import { z } from 'zod'; +import { completeStructured, type CompleteFn } from '../complete.js'; +import { DEFAULT_OLLAMA_BASE_URL, DEFAULT_OLLAMA_MODEL } from '../config.js'; +import { buildClipPrompt, buildSummaryPrompt } from '../prompts.js'; +import { clipCandidatesSchema, sessionNoteSchema, type ClipCandidate, type SessionNote } from '../schemas.js'; +import type { AiProvider, SelectClipsOptions, SummarizeOptions, TranscriptInput } from '../types.js'; + +/** Minimal response surface `ai-core` needs from a fetch implementation. */ +export interface FetchResponseLike { + readonly ok: boolean; + readonly status: number; + readonly statusText: string; + json(): Promise; +} + +/** Injectable fetch, so the fully-local Ollama path is testable without a network. */ +export type FetchLike = ( + url: string, + init: { method: string; headers: Record; body: string }, +) => Promise; + +const ollamaChatResponseSchema = z.object({ + message: z.object({ content: z.string() }), +}); + +export interface OllamaProviderOptions { + model?: string; + baseUrl?: string; + fetchImpl?: FetchLike; +} + +const defaultFetch: FetchLike = (url, init) => (globalThis as unknown as { fetch: FetchLike }).fetch(url, init); + +/** Fully-local provider (nothing leaves the device) backed by an Ollama server. */ +export function createOllamaProvider(options: OllamaProviderOptions = {}): AiProvider { + const model = options.model ?? DEFAULT_OLLAMA_MODEL; + const baseUrl = (options.baseUrl ?? DEFAULT_OLLAMA_BASE_URL).replace(/\/+$/, ''); + const fetchImpl = options.fetchImpl ?? defaultFetch; + + const complete: CompleteFn = async ({ system, user }) => { + const response = await fetchImpl(`${baseUrl}/api/chat`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + model, + stream: false, + messages: [ + { role: 'system', content: system }, + { role: 'user', content: user }, + ], + }), + }); + if (!response.ok) { + throw new Error(`Ollama request failed: ${String(response.status)} ${response.statusText}`); + } + const data = ollamaChatResponseSchema.parse(await response.json()); + return data.message.content; + }; + + return { + kind: 'ollama', + model, + summarizeSession(input: TranscriptInput, summarizeOptions?: SummarizeOptions): Promise { + return completeStructured(complete, buildSummaryPrompt(input, summarizeOptions), sessionNoteSchema); + }, + selectClips(input: TranscriptInput, selectOptions?: SelectClipsOptions): Promise { + return completeStructured(complete, buildClipPrompt(input, selectOptions), clipCandidatesSchema); + }, + }; +} diff --git a/packages/ai-core/src/providers/providers.test.ts b/packages/ai-core/src/providers/providers.test.ts new file mode 100644 index 00000000..3af24438 --- /dev/null +++ b/packages/ai-core/src/providers/providers.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it, vi } from 'vitest'; +import { createProvider } from './factory.js'; +import { + createAnthropicProvider, + type AnthropicClientLike, + type AnthropicCreateBody, + type AnthropicMessageResponse, +} from './anthropic.js'; +import { createOllamaProvider, type FetchLike, type FetchResponseLike } from './ollama.js'; +import type { TranscriptInput } from '../types.js'; + +const input: TranscriptInput = { + segments: [{ startMs: 0, endMs: 3000, speaker: 'Host', text: 'ship it' }], +}; + +const NOTE_JSON = JSON.stringify({ + tldr: 'ship it', + decisions: [], + actionItems: [], + topics: ['release'], +}); + +const CLIPS_JSON = JSON.stringify([ + { startMs: 0, endMs: 3000, title: 'Ship it', hookCaption: 'We shipped', platformFit: ['shorts'] }, +]); + +function fakeAnthropic(replies: string[]): { client: AnthropicClientLike; bodies: AnthropicCreateBody[] } { + const bodies: AnthropicCreateBody[] = []; + let call = 0; + const client: AnthropicClientLike = { + messages: { + create(body: AnthropicCreateBody): Promise { + bodies.push(body); + const text = replies[call] ?? replies[replies.length - 1] ?? ''; + call += 1; + return Promise.resolve({ content: [{ type: 'text', text }] }); + }, + }, + }; + return { client, bodies }; +} + +describe('anthropic provider', () => { + it('defaults to the Haiku model (finding F3)', () => { + const { client } = fakeAnthropic([NOTE_JSON]); + expect(createAnthropicProvider({ client }).model).toBe('claude-haiku-4-5'); + }); + + it('summarizes and sends only transcript text', async () => { + const { client, bodies } = fakeAnthropic([NOTE_JSON]); + const note = await createAnthropicProvider({ client, model: 'claude-sonnet-5' }).summarizeSession(input); + expect(note.topics).toEqual(['release']); + expect(bodies[0]?.model).toBe('claude-sonnet-5'); + expect(bodies[0]?.messages[0]?.content).toContain('ship it'); + }); + + it('selects clips and validates them', async () => { + const { client } = fakeAnthropic([CLIPS_JSON]); + const clips = await createAnthropicProvider({ client }).selectClips(input, { maxClips: 1 }); + expect(clips).toHaveLength(1); + expect(clips[0]?.title).toBe('Ship it'); + }); + + it('repairs one malformed reply before succeeding', async () => { + const { client, bodies } = fakeAnthropic(['not json at all', NOTE_JSON]); + const note = await createAnthropicProvider({ client }).summarizeSession(input); + expect(note.tldr).toBe('ship it'); + expect(bodies).toHaveLength(2); + }); +}); + +function fakeFetch(content: string): { fetchImpl: FetchLike; urls: string[] } { + const urls: string[] = []; + const fetchImpl: FetchLike = (url) => { + urls.push(url); + const response: FetchResponseLike = { + ok: true, + status: 200, + statusText: 'OK', + json: () => Promise.resolve({ message: { content } }), + }; + return Promise.resolve(response); + }; + return { fetchImpl, urls }; +} + +describe('ollama provider', () => { + it('posts to /api/chat and parses the reply', async () => { + const { fetchImpl, urls } = fakeFetch(NOTE_JSON); + const note = await createOllamaProvider({ fetchImpl, baseUrl: 'http://localhost:11434/' }).summarizeSession(input); + expect(note.topics).toEqual(['release']); + expect(urls[0]).toBe('http://localhost:11434/api/chat'); + }); + + it('throws on a non-ok response', async () => { + const fetchImpl: FetchLike = () => + Promise.resolve({ ok: false, status: 500, statusText: 'Server Error', json: () => Promise.resolve({}) }); + await expect(createOllamaProvider({ fetchImpl }).summarizeSession(input)).rejects.toThrow('Ollama request failed'); + }); +}); + +describe('createProvider', () => { + it('builds an anthropic provider', () => { + const { client } = fakeAnthropic([NOTE_JSON]); + expect(createProvider({ kind: 'anthropic', client }).kind).toBe('anthropic'); + }); + + it('builds an ollama provider', () => { + expect(createProvider({ kind: 'ollama', fetchImpl: vi.fn() }).kind).toBe('ollama'); + }); +}); diff --git a/packages/ai-core/src/schemas.test.ts b/packages/ai-core/src/schemas.test.ts new file mode 100644 index 00000000..a2f91cfa --- /dev/null +++ b/packages/ai-core/src/schemas.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; +import { clipCandidateSchema, clipCandidatesSchema, sessionNoteSchema } from './schemas.js'; + +describe('sessionNoteSchema', () => { + it('accepts a well-formed note', () => { + const result = sessionNoteSchema.safeParse({ + tldr: 'We fixed the flaky auth test.', + decisions: ['Pin the clock in tests'], + actionItems: [{ description: 'Backport the fix', owner: 'Ada', deadline: 'Friday', timestampMs: 120000 }], + topics: ['auth', 'testing'], + }); + expect(result.success).toBe(true); + }); + + it('allows nullable owner/deadline/timestamp', () => { + const result = sessionNoteSchema.safeParse({ + tldr: 'x', + decisions: [], + actionItems: [{ description: 'Do the thing', owner: null, deadline: null, timestampMs: null }], + topics: [], + }); + expect(result.success).toBe(true); + }); + + it('rejects an empty tldr', () => { + const result = sessionNoteSchema.safeParse({ tldr: '', decisions: [], actionItems: [], topics: [] }); + expect(result.success).toBe(false); + }); +}); + +describe('clipCandidateSchema', () => { + it('rejects a clip whose end is not after its start', () => { + const result = clipCandidateSchema.safeParse({ + startMs: 5000, + endMs: 5000, + title: 'Nope', + hookCaption: 'Nope', + platformFit: ['tiktok'], + }); + expect(result.success).toBe(false); + }); + + it('rejects an unknown platform', () => { + const result = clipCandidateSchema.safeParse({ + startMs: 0, + endMs: 1000, + title: 'Hi', + hookCaption: 'Hi', + platformFit: ['myspace'], + }); + expect(result.success).toBe(false); + }); +}); + +describe('clipCandidatesSchema', () => { + const clip = { startMs: 0, endMs: 1000, title: 't', hookCaption: 'h', platformFit: ['shorts'] as const }; + + it('rejects an empty array', () => { + expect(clipCandidatesSchema.safeParse([]).success).toBe(false); + }); + + it('rejects more than eight candidates', () => { + const nine = Array.from({ length: 9 }, () => clip); + expect(clipCandidatesSchema.safeParse(nine).success).toBe(false); + }); + + it('accepts a valid batch', () => { + expect(clipCandidatesSchema.safeParse([clip, clip]).success).toBe(true); + }); +}); diff --git a/packages/ai-core/src/schemas.ts b/packages/ai-core/src/schemas.ts new file mode 100644 index 00000000..d3e1aa91 --- /dev/null +++ b/packages/ai-core/src/schemas.ts @@ -0,0 +1,51 @@ +/** + * Structured-output schemas for AI Session Notes and AI Clips. + * + * These are the single source of truth for the shapes an LLM provider must + * return. Every provider response is validated against them before it reaches + * the rest of the app, so malformed model output can never leak downstream. + */ +import { z } from 'zod'; + +/** One TODO extracted from a session: what, who owns it, when, and where in the recording. */ +export const actionItemSchema = z.object({ + description: z.string().min(1), + /** Assignee display name, or null if the transcript didn't name one. */ + owner: z.string().nullable(), + /** Deadline as stated in the session (free text), or null. */ + deadline: z.string().nullable(), + /** Deep-link offset into the recording, in milliseconds, or null. */ + timestampMs: z.number().int().nonnegative().nullable(), +}); +export type ActionItem = z.infer; + +/** Structured notes produced from a session transcript. */ +export const sessionNoteSchema = z.object({ + tldr: z.string().min(1), + decisions: z.array(z.string()), + actionItems: z.array(actionItemSchema), + topics: z.array(z.string()), +}); +export type SessionNote = z.infer; + +/** Social destinations a clip is a good fit for. */ +export const platformFitSchema = z.enum(['shorts', 'tiktok', 'reels', 'x', 'linkedin', 'youtube']); +export type PlatformFit = z.infer; + +/** A single highlight candidate the model proposes for clipping. */ +export const clipCandidateSchema = z + .object({ + startMs: z.number().int().nonnegative(), + endMs: z.number().int().positive(), + title: z.string().min(1), + hookCaption: z.string().min(1), + platformFit: z.array(platformFitSchema), + }) + .refine((clip) => clip.endMs > clip.startMs, { + message: 'endMs must be greater than startMs', + path: ['endMs'], + }); +export type ClipCandidate = z.infer; + +/** The array shape a clip-selection call must return (3–8 candidates, per the PRD). */ +export const clipCandidatesSchema = z.array(clipCandidateSchema).min(1).max(8); diff --git a/packages/ai-core/src/templates.ts b/packages/ai-core/src/templates.ts new file mode 100644 index 00000000..6713cb2e --- /dev/null +++ b/packages/ai-core/src/templates.ts @@ -0,0 +1,35 @@ +/** + * Built-in summary templates. The PRD ships three at launch; the guidance text + * is injected into the summarization system prompt to steer what the notes emphasize. + */ +export const SESSION_TEMPLATE_IDS = ['pair-programming', 'support-session', 'user-interview'] as const; +export type SessionTemplateId = (typeof SESSION_TEMPLATE_IDS)[number]; + +export interface SessionTemplateSpec { + readonly label: string; + readonly guidance: string; +} + +export const SESSION_TEMPLATES: Readonly> = { + 'pair-programming': { + label: 'Pair Programming', + guidance: + 'Focus on technical decisions, the approaches chosen and rejected, bugs found and fixed, and concrete TODOs with the engineer responsible.', + }, + 'support-session': { + label: 'Support Session', + guidance: + 'Focus on the reported problem, the diagnostic steps taken, the resolution applied, and any follow-up owed to the customer.', + }, + 'user-interview': { + label: 'User Interview', + guidance: + 'Focus on the user’s goals and pain points, notable quoted reactions, feature requests, and hypotheses to follow up on.', + }, +}; + +export const DEFAULT_TEMPLATE: SessionTemplateId = 'pair-programming'; + +export function getTemplate(id: SessionTemplateId | undefined): SessionTemplateSpec { + return SESSION_TEMPLATES[id ?? DEFAULT_TEMPLATE]; +} diff --git a/packages/ai-core/src/transcript.test.ts b/packages/ai-core/src/transcript.test.ts new file mode 100644 index 00000000..d6babbd1 --- /dev/null +++ b/packages/ai-core/src/transcript.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest'; +import { formatSegment, formatTimestamp, formatTranscript } from './transcript.js'; + +describe('formatTimestamp', () => { + it('formats sub-minute and multi-minute offsets', () => { + expect(formatTimestamp(0)).toBe('00:00'); + expect(formatTimestamp(9000)).toBe('00:09'); + expect(formatTimestamp(125000)).toBe('02:05'); + }); + + it('clamps negative input to zero', () => { + expect(formatTimestamp(-500)).toBe('00:00'); + }); +}); + +describe('formatSegment / formatTranscript', () => { + it('renders one segment', () => { + expect(formatSegment({ startMs: 65000, endMs: 70000, speaker: 'Host', text: 'hi' })).toBe('[01:05 Host] hi'); + }); + + it('joins segments with newlines', () => { + const text = formatTranscript({ + segments: [ + { startMs: 0, endMs: 1000, speaker: 'A', text: 'one' }, + { startMs: 1000, endMs: 2000, speaker: 'B', text: 'two' }, + ], + }); + expect(text).toBe('[00:00 A] one\n[00:01 B] two'); + }); +}); diff --git a/packages/ai-core/src/transcript.ts b/packages/ai-core/src/transcript.ts new file mode 100644 index 00000000..11e452f1 --- /dev/null +++ b/packages/ai-core/src/transcript.ts @@ -0,0 +1,19 @@ +import type { TranscriptInput, TranscriptSegment } from './types.js'; + +/** Format a millisecond offset as `mm:ss` (minutes are not zero-capped at 60). */ +export function formatTimestamp(ms: number): string { + const totalSeconds = Math.max(0, Math.floor(ms / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return `${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')}`; +} + +/** Render one segment as `[mm:ss Speaker] text`. */ +export function formatSegment(segment: TranscriptSegment): string { + return `[${formatTimestamp(segment.startMs)} ${segment.speaker}] ${segment.text}`; +} + +/** Render the whole transcript, one segment per line, for prompt inclusion. */ +export function formatTranscript(input: TranscriptInput): string { + return input.segments.map((segment) => formatSegment(segment)).join('\n'); +} diff --git a/packages/ai-core/src/types.ts b/packages/ai-core/src/types.ts new file mode 100644 index 00000000..7daefa85 --- /dev/null +++ b/packages/ai-core/src/types.ts @@ -0,0 +1,49 @@ +import type { ClipCandidate, PlatformFit, SessionNote } from './schemas.js'; +import type { SessionTemplateId } from './templates.js'; + +/** One attributed line of transcript. Track-level speaker labels ("Host", "Viewer-1"). */ +export interface TranscriptSegment { + /** Millisecond offset from session start. */ + startMs: number; + endMs: number; + speaker: string; + text: string; +} + +/** + * The only thing ever handed to a provider. + * + * By construction this carries transcript TEXT and lightweight metadata only — + * never audio or video. That is the boundary that keeps the "media never leaves + * the device" promise intact even in managed-key mode. + */ +export interface TranscriptInput { + segments: TranscriptSegment[]; + /** Optional scene-change offsets (ms) from a cheap ffmpeg pass, used to bias clip selection. */ + sceneChangesMs?: number[]; + /** Human-readable session title/context. */ + title?: string; + /** Total session duration in ms, if known. */ + durationMs?: number; +} + +export type ProviderKind = 'anthropic' | 'ollama'; + +export interface SummarizeOptions { + template?: SessionTemplateId; +} + +export interface SelectClipsOptions { + /** Bias selection toward a target social platform. */ + platform?: PlatformFit; + /** Cap the number of candidates requested (clamped to the schema's 1–8 range). */ + maxClips?: number; +} + +/** A configured provider. Both methods send transcript text only and return validated output. */ +export interface AiProvider { + readonly kind: ProviderKind; + readonly model: string; + summarizeSession(input: TranscriptInput, options?: SummarizeOptions): Promise; + selectClips(input: TranscriptInput, options?: SelectClipsOptions): Promise; +} diff --git a/packages/ai-core/tsconfig.json b/packages/ai-core/tsconfig.json new file mode 100644 index 00000000..77bca1f3 --- /dev/null +++ b/packages/ai-core/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src", + "declaration": true, + "declarationMap": true, + "noEmit": false + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "src/**/*.test.ts"] +} diff --git a/packages/ai-core/vitest.config.ts b/packages/ai-core/vitest.config.ts new file mode 100644 index 00000000..7dd13254 --- /dev/null +++ b/packages/ai-core/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + include: ['src/**/*.test.ts'], + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 25eb2fdb..21654e69 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -432,6 +432,19 @@ importers: specifier: ^3.2.0 version: 3.2.6(@types/debug@4.1.12)(@types/node@22.19.7)(jsdom@25.0.1)(lightningcss@1.30.2)(terser@5.46.0) + packages/ai-core: + dependencies: + zod: + specifier: ^3.24.0 + version: 3.25.76 + devDependencies: + typescript: + specifier: ^5.7.0 + version: 5.9.3 + vitest: + specifier: ^3.2.0 + version: 3.2.6(@types/debug@4.1.12)(@types/node@24.10.9)(jsdom@25.0.1)(lightningcss@1.30.2)(terser@5.46.0) + packages/shared-types: devDependencies: typescript: @@ -1730,7 +1743,7 @@ packages: '@expo/bunyan@4.0.1': resolution: {integrity: sha512-+Lla7nYSiHZirgK+U/uYzsLv/X+HaJienbD5AKX1UQZHYfWaP+9uuQluRB4GrEVWF0GZ7vEVp/jzaOT9k/SQlg==} - engines: {node: '>=0.10.0'} + engines: {'0': node >=0.10.0} '@expo/cli@0.22.27': resolution: {integrity: sha512-MZ3s68+OFQZWljAiCdwL+dL+xZudpkmhq0A2Qb4+p6MNp386WJyAYineXtiLJVww/8ohIfUxOL10vnaPdDVl4w==} @@ -2384,19 +2397,16 @@ packages: '@nut-tree-fork/libnut-darwin@2.7.5': resolution: {integrity: sha512-LbqtPtMPTJUcg4XoPP2jsU1wc8flBcGyKTerKsIfK9cD7nBHROnO0QksbrsbSWEpLym8T8fRtuU7XEY83l6Z2Q==} engines: {node: '>=10.15.3'} - cpu: [x64, arm64] os: [darwin, linux, win32] '@nut-tree-fork/libnut-linux@2.7.5': resolution: {integrity: sha512-uxaXEcRKnFObAljsoR6tLOBUU1dJ2sctloG6gFgCBGN7+k6Jdv6jZfOuNjd/fpdq2C5WPMm0rtn9EE7h5J3Jcg==} engines: {node: '>=10.15.3'} - cpu: [x64, arm64] os: [darwin, linux, win32] '@nut-tree-fork/libnut-win32@2.7.5': resolution: {integrity: sha512-yqC87zvmFcDPwFrRU40DYhN0xmEVM3aSkOuyF0IX+y1x+HWSu/i0PNklATpPBhGid3QVb/TOHuVoaraMrUFCNw==} engines: {node: '>=10.15.3'} - cpu: [x64, arm64] os: [darwin, linux, win32] '@nut-tree-fork/libnut@4.2.6': @@ -2410,7 +2420,6 @@ packages: '@nut-tree-fork/nut-js@4.2.6': resolution: {integrity: sha512-aI/WCX7gE1HFGPH3EZP/UWqpNMM1NMoM/EkXqp7pKMgXFCi8e5+o5p+jd/QOYpmALv9bQg7+s69nI7FONbMqDg==} engines: {node: '>=16'} - cpu: [x64, arm64] os: [linux, darwin, win32] '@nut-tree-fork/provider-interfaces@4.2.6': @@ -8414,7 +8423,7 @@ snapshots: '@babel/helper-member-expression-to-functions@7.28.5': dependencies: '@babel/traverse': 7.28.6 - '@babel/types': 7.28.6 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color @@ -8452,7 +8461,7 @@ snapshots: '@babel/helper-optimise-call-expression@7.27.1': dependencies: - '@babel/types': 7.28.6 + '@babel/types': 7.29.7 '@babel/helper-plugin-utils@7.28.6': {} @@ -8508,7 +8517,7 @@ snapshots: '@babel/highlight@7.25.9': dependencies: - '@babel/helper-validator-identifier': 7.28.5 + '@babel/helper-validator-identifier': 7.29.7 chalk: 2.4.2 js-tokens: 4.0.0 picocolors: 1.1.1 @@ -8778,7 +8787,7 @@ snapshots: dependencies: '@babel/core': 7.28.6 '@babel/helper-plugin-utils': 7.28.6 - '@babel/template': 7.28.6 + '@babel/template': 7.29.7 '@babel/plugin-transform-destructuring@7.28.5(@babel/core@7.28.6)': dependencies: @@ -9014,7 +9023,7 @@ snapshots: '@babel/helper-module-imports': 7.28.6 '@babel/helper-plugin-utils': 7.28.6 '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.28.6) - '@babel/types': 7.28.6 + '@babel/types': 7.29.7 transitivePeerDependencies: - supports-color @@ -12061,8 +12070,8 @@ snapshots: babel-plugin-jest-hoist@29.6.3: dependencies: - '@babel/template': 7.28.6 - '@babel/types': 7.28.6 + '@babel/template': 7.29.7 + '@babel/types': 7.29.7 '@types/babel__core': 7.20.5 '@types/babel__traverse': 7.28.0 @@ -14299,7 +14308,7 @@ snapshots: istanbul-lib-instrument@5.2.1: dependencies: '@babel/core': 7.28.6 - '@babel/parser': 7.28.6 + '@babel/parser': 7.29.7 '@istanbuljs/schema': 0.1.3 istanbul-lib-coverage: 3.2.2 semver: 6.3.1 @@ -14375,7 +14384,7 @@ snapshots: jest-message-util@29.7.0: dependencies: - '@babel/code-frame': 7.28.6 + '@babel/code-frame': 7.29.7 '@jest/types': 29.6.3 '@types/stack-utils': 2.0.3 chalk: 4.1.2 @@ -14987,8 +14996,8 @@ snapshots: metro-transform-plugins@0.81.5: dependencies: '@babel/core': 7.28.6 - '@babel/generator': 7.28.6 - '@babel/template': 7.28.6 + '@babel/generator': 7.29.7 + '@babel/template': 7.29.7 '@babel/traverse': 7.28.6 flow-enums-runtime: 0.0.6 nullthrows: 1.1.1 @@ -14998,9 +15007,9 @@ snapshots: metro-transform-worker@0.81.5: dependencies: '@babel/core': 7.28.6 - '@babel/generator': 7.28.6 - '@babel/parser': 7.28.6 - '@babel/types': 7.28.6 + '@babel/generator': 7.29.7 + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 flow-enums-runtime: 0.0.6 metro: 0.81.5 metro-babel-transformer: 0.81.5 From 0e705e75131242233ef67c12dc02fbda159d352d Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 5 Jul 2026 01:30:30 +0000 Subject: [PATCH 2/2] fix(ai-core): resolve CodeQL ReDoS alerts and Prettier formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI "Lint" job failed on `pnpm format:check`; CodeQL flagged two ReDoS-prone regexes. Both are in this package. - parse.ts: replace the ```-fence and first-bracket regexes with linear index scans (indexOf / char loop) — no backtracking on adversarial model output - providers/ollama.ts: strip trailing slashes with a char loop instead of /\/+$/ - run Prettier over the package (printWidth 100, es5 trailing commas) to satisfy format:check No behavior change; 37 tests still pass, typecheck + eslint clean. Co-Authored-By: Claude Opus 4.8 --- packages/ai-core/src/complete.test.ts | 15 +++-- packages/ai-core/src/complete.ts | 6 +- packages/ai-core/src/parse.ts | 64 +++++++++++++++---- packages/ai-core/src/prompts.ts | 5 +- packages/ai-core/src/providers/anthropic.ts | 40 ++++++++++-- packages/ai-core/src/providers/ollama.ts | 52 ++++++++++++--- .../ai-core/src/providers/providers.test.ts | 26 ++++++-- packages/ai-core/src/schemas.test.ts | 23 +++++-- packages/ai-core/src/templates.ts | 6 +- packages/ai-core/src/transcript.test.ts | 4 +- 10 files changed, 193 insertions(+), 48 deletions(-) diff --git a/packages/ai-core/src/complete.test.ts b/packages/ai-core/src/complete.test.ts index b2a35c9f..2d2d0087 100644 --- a/packages/ai-core/src/complete.test.ts +++ b/packages/ai-core/src/complete.test.ts @@ -8,7 +8,9 @@ const schema = z.object({ ok: z.boolean() }); describe('completeStructured', () => { it('returns parsed output when the first reply is valid', async () => { const complete: CompleteFn = vi.fn(() => Promise.resolve('{"ok": true}')); - await expect(completeStructured(complete, { system: 's', user: 'u' }, schema)).resolves.toEqual({ ok: true }); + await expect(completeStructured(complete, { system: 's', user: 'u' }, schema)).resolves.toEqual( + { ok: true } + ); expect(complete).toHaveBeenCalledTimes(1); }); @@ -23,7 +25,10 @@ describe('completeStructured', () => { }); it('asks for JSON-only on the repair attempt', async () => { - const complete = vi.fn().mockResolvedValueOnce('nope').mockResolvedValueOnce('{"ok": true}'); + const complete = vi + .fn() + .mockResolvedValueOnce('nope') + .mockResolvedValueOnce('{"ok": true}'); await completeStructured(complete, { system: 's', user: 'u' }, schema); const secondCall = complete.mock.calls[1]?.[0]; expect(secondCall?.user).toContain('ONLY the JSON'); @@ -32,9 +37,9 @@ describe('completeStructured', () => { it('rethrows StructuredOutputError after a second failure (caller falls back to manual mode)', async () => { const complete: CompleteFn = vi.fn(() => Promise.resolve('still not json')); - await expect(completeStructured(complete, { system: 's', user: 'u' }, schema)).rejects.toBeInstanceOf( - StructuredOutputError, - ); + await expect( + completeStructured(complete, { system: 's', user: 'u' }, schema) + ).rejects.toBeInstanceOf(StructuredOutputError); expect(complete).toHaveBeenCalledTimes(2); }); }); diff --git a/packages/ai-core/src/complete.ts b/packages/ai-core/src/complete.ts index 1a962b29..a02ae583 100644 --- a/packages/ai-core/src/complete.ts +++ b/packages/ai-core/src/complete.ts @@ -17,7 +17,11 @@ export type CompleteFn = (prompt: Prompt) => Promise; * auto-repair retry (re-asking for JSON only); a second failure rethrows the * {@link StructuredOutputError} so the caller can fall back to manual mode. */ -export async function completeStructured(complete: CompleteFn, prompt: Prompt, schema: ZodType): Promise { +export async function completeStructured( + complete: CompleteFn, + prompt: Prompt, + schema: ZodType +): Promise { const first = await complete(prompt); try { return parseStructured(first, schema); diff --git a/packages/ai-core/src/parse.ts b/packages/ai-core/src/parse.ts index ca636b4f..d39587a4 100644 --- a/packages/ai-core/src/parse.ts +++ b/packages/ai-core/src/parse.ts @@ -1,28 +1,60 @@ import type { ZodType } from 'zod'; import { StructuredOutputError } from './errors.js'; +const FENCE = '```'; + +/** True if `text` is a short run of ASCII letters (a code-fence language tag like "json"). */ +function isLanguageTag(text: string): boolean { + if (text.length === 0 || text.length > 16) { + return false; + } + for (const char of text) { + const isLetter = (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z'); + if (!isLetter) { + return false; + } + } + return true; +} + +/** Index of the first `{` or `[` in `raw`, or -1. Linear scan (no regex, no backtracking). */ +function firstBracket(raw: string): number { + for (let i = 0; i < raw.length; i += 1) { + const char = raw[i]; + if (char === '{' || char === '[') { + return i; + } + } + return -1; +} + /** * Pull the first JSON value out of a model reply that may wrap it in prose or a - * ```json fence. Best-effort: returns the most plausible JSON substring, which - * `parseStructured` then validates. + * ```json fence. Uses index scans only (no regex) so it cannot backtrack on + * adversarial input. Best-effort: returns the most plausible JSON substring, + * which `parseStructured` then validates. */ export function extractJson(raw: string): string { - const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(raw); - const fencedBody = fenced?.[1]; - if (fencedBody !== undefined) { - return fencedBody.trim(); + const fenceStart = raw.indexOf(FENCE); + if (fenceStart !== -1) { + const afterOpen = fenceStart + FENCE.length; + const fenceEnd = raw.indexOf(FENCE, afterOpen); + if (fenceEnd !== -1) { + let body = raw.slice(afterOpen, fenceEnd); + const newline = body.indexOf('\n'); + if (newline !== -1 && isLanguageTag(body.slice(0, newline).trim())) { + body = body.slice(newline + 1); + } + return body.trim(); + } } - const start = raw.search(/[[{]/); + const start = firstBracket(raw); if (start === -1) { return raw.trim(); } const opener = raw[start]; - if (opener === undefined) { - return raw.trim(); - } - const closer = opener === '[' ? ']' : '}'; const end = raw.lastIndexOf(closer); if (end > start) { @@ -44,9 +76,13 @@ export function parseStructured(raw: string, schema: ZodType): T { const result = schema.safeParse(parsed); if (!result.success) { - throw new StructuredOutputError(`model output failed schema validation: ${result.error.message}`, raw, { - cause: result.error, - }); + throw new StructuredOutputError( + `model output failed schema validation: ${result.error.message}`, + raw, + { + cause: result.error, + } + ); } return result.data; } diff --git a/packages/ai-core/src/prompts.ts b/packages/ai-core/src/prompts.ts index 27032aed..fce5aae3 100644 --- a/packages/ai-core/src/prompts.ts +++ b/packages/ai-core/src/prompts.ts @@ -25,7 +25,10 @@ export function buildSummaryPrompt(input: TranscriptInput, options?: SummarizeOp /** Build the clip-selection prompt: highlight instructions + optional scene changes. */ export function buildClipPrompt(input: TranscriptInput, options?: SelectClipsOptions): Prompt { const maxClips = options?.maxClips ?? DEFAULT_MAX_CLIPS; - const platformLine = options?.platform !== undefined ? ` Bias selection toward content that performs well on ${options.platform}.` : ''; + const platformLine = + options?.platform !== undefined + ? ` Bias selection toward content that performs well on ${options.platform}.` + : ''; const sceneLine = input.sceneChangesMs !== undefined && input.sceneChangesMs.length > 0 ? `\n\nScene-change timestamps (ms): ${input.sceneChangesMs.join(', ')}` diff --git a/packages/ai-core/src/providers/anthropic.ts b/packages/ai-core/src/providers/anthropic.ts index de577df2..2e135416 100644 --- a/packages/ai-core/src/providers/anthropic.ts +++ b/packages/ai-core/src/providers/anthropic.ts @@ -1,8 +1,18 @@ import { completeStructured, type CompleteFn } from '../complete.js'; import { DEFAULT_ANTHROPIC_MODEL, DEFAULT_MAX_TOKENS } from '../config.js'; import { buildClipPrompt, buildSummaryPrompt } from '../prompts.js'; -import { clipCandidatesSchema, sessionNoteSchema, type ClipCandidate, type SessionNote } from '../schemas.js'; -import type { AiProvider, SelectClipsOptions, SummarizeOptions, TranscriptInput } from '../types.js'; +import { + clipCandidatesSchema, + sessionNoteSchema, + type ClipCandidate, + type SessionNote, +} from '../schemas.js'; +import type { + AiProvider, + SelectClipsOptions, + SummarizeOptions, + TranscriptInput, +} from '../types.js'; /** * Structural view of the Anthropic Messages client. `ai-core` does not depend on @@ -53,17 +63,33 @@ export function createAnthropicProvider(options: AnthropicProviderOptions): AiPr system, messages: [{ role: 'user', content: user }], }); - return response.content.map((block) => (block.type === 'text' ? (block.text ?? '') : '')).join(''); + return response.content + .map((block) => (block.type === 'text' ? (block.text ?? '') : '')) + .join(''); }; return { kind: 'anthropic', model, - summarizeSession(input: TranscriptInput, summarizeOptions?: SummarizeOptions): Promise { - return completeStructured(complete, buildSummaryPrompt(input, summarizeOptions), sessionNoteSchema); + summarizeSession( + input: TranscriptInput, + summarizeOptions?: SummarizeOptions + ): Promise { + return completeStructured( + complete, + buildSummaryPrompt(input, summarizeOptions), + sessionNoteSchema + ); }, - selectClips(input: TranscriptInput, selectOptions?: SelectClipsOptions): Promise { - return completeStructured(complete, buildClipPrompt(input, selectOptions), clipCandidatesSchema); + selectClips( + input: TranscriptInput, + selectOptions?: SelectClipsOptions + ): Promise { + return completeStructured( + complete, + buildClipPrompt(input, selectOptions), + clipCandidatesSchema + ); }, }; } diff --git a/packages/ai-core/src/providers/ollama.ts b/packages/ai-core/src/providers/ollama.ts index ac1f63bc..ce4653c6 100644 --- a/packages/ai-core/src/providers/ollama.ts +++ b/packages/ai-core/src/providers/ollama.ts @@ -2,8 +2,18 @@ import { z } from 'zod'; import { completeStructured, type CompleteFn } from '../complete.js'; import { DEFAULT_OLLAMA_BASE_URL, DEFAULT_OLLAMA_MODEL } from '../config.js'; import { buildClipPrompt, buildSummaryPrompt } from '../prompts.js'; -import { clipCandidatesSchema, sessionNoteSchema, type ClipCandidate, type SessionNote } from '../schemas.js'; -import type { AiProvider, SelectClipsOptions, SummarizeOptions, TranscriptInput } from '../types.js'; +import { + clipCandidatesSchema, + sessionNoteSchema, + type ClipCandidate, + type SessionNote, +} from '../schemas.js'; +import type { + AiProvider, + SelectClipsOptions, + SummarizeOptions, + TranscriptInput, +} from '../types.js'; /** Minimal response surface `ai-core` needs from a fetch implementation. */ export interface FetchResponseLike { @@ -16,7 +26,7 @@ export interface FetchResponseLike { /** Injectable fetch, so the fully-local Ollama path is testable without a network. */ export type FetchLike = ( url: string, - init: { method: string; headers: Record; body: string }, + init: { method: string; headers: Record; body: string } ) => Promise; const ollamaChatResponseSchema = z.object({ @@ -29,12 +39,22 @@ export interface OllamaProviderOptions { fetchImpl?: FetchLike; } -const defaultFetch: FetchLike = (url, init) => (globalThis as unknown as { fetch: FetchLike }).fetch(url, init); +const defaultFetch: FetchLike = (url, init) => + (globalThis as unknown as { fetch: FetchLike }).fetch(url, init); + +/** Strip trailing slashes without a regex (avoids ReDoS on adversarial base URLs). */ +function stripTrailingSlashes(url: string): string { + let end = url.length; + while (end > 0 && url[end - 1] === '/') { + end -= 1; + } + return url.slice(0, end); +} /** Fully-local provider (nothing leaves the device) backed by an Ollama server. */ export function createOllamaProvider(options: OllamaProviderOptions = {}): AiProvider { const model = options.model ?? DEFAULT_OLLAMA_MODEL; - const baseUrl = (options.baseUrl ?? DEFAULT_OLLAMA_BASE_URL).replace(/\/+$/, ''); + const baseUrl = stripTrailingSlashes(options.baseUrl ?? DEFAULT_OLLAMA_BASE_URL); const fetchImpl = options.fetchImpl ?? defaultFetch; const complete: CompleteFn = async ({ system, user }) => { @@ -60,11 +80,25 @@ export function createOllamaProvider(options: OllamaProviderOptions = {}): AiPro return { kind: 'ollama', model, - summarizeSession(input: TranscriptInput, summarizeOptions?: SummarizeOptions): Promise { - return completeStructured(complete, buildSummaryPrompt(input, summarizeOptions), sessionNoteSchema); + summarizeSession( + input: TranscriptInput, + summarizeOptions?: SummarizeOptions + ): Promise { + return completeStructured( + complete, + buildSummaryPrompt(input, summarizeOptions), + sessionNoteSchema + ); }, - selectClips(input: TranscriptInput, selectOptions?: SelectClipsOptions): Promise { - return completeStructured(complete, buildClipPrompt(input, selectOptions), clipCandidatesSchema); + selectClips( + input: TranscriptInput, + selectOptions?: SelectClipsOptions + ): Promise { + return completeStructured( + complete, + buildClipPrompt(input, selectOptions), + clipCandidatesSchema + ); }, }; } diff --git a/packages/ai-core/src/providers/providers.test.ts b/packages/ai-core/src/providers/providers.test.ts index 3af24438..6553edf2 100644 --- a/packages/ai-core/src/providers/providers.test.ts +++ b/packages/ai-core/src/providers/providers.test.ts @@ -24,7 +24,10 @@ const CLIPS_JSON = JSON.stringify([ { startMs: 0, endMs: 3000, title: 'Ship it', hookCaption: 'We shipped', platformFit: ['shorts'] }, ]); -function fakeAnthropic(replies: string[]): { client: AnthropicClientLike; bodies: AnthropicCreateBody[] } { +function fakeAnthropic(replies: string[]): { + client: AnthropicClientLike; + bodies: AnthropicCreateBody[]; +} { const bodies: AnthropicCreateBody[] = []; let call = 0; const client: AnthropicClientLike = { @@ -48,7 +51,10 @@ describe('anthropic provider', () => { it('summarizes and sends only transcript text', async () => { const { client, bodies } = fakeAnthropic([NOTE_JSON]); - const note = await createAnthropicProvider({ client, model: 'claude-sonnet-5' }).summarizeSession(input); + const note = await createAnthropicProvider({ + client, + model: 'claude-sonnet-5', + }).summarizeSession(input); expect(note.topics).toEqual(['release']); expect(bodies[0]?.model).toBe('claude-sonnet-5'); expect(bodies[0]?.messages[0]?.content).toContain('ship it'); @@ -87,15 +93,25 @@ function fakeFetch(content: string): { fetchImpl: FetchLike; urls: string[] } { describe('ollama provider', () => { it('posts to /api/chat and parses the reply', async () => { const { fetchImpl, urls } = fakeFetch(NOTE_JSON); - const note = await createOllamaProvider({ fetchImpl, baseUrl: 'http://localhost:11434/' }).summarizeSession(input); + const note = await createOllamaProvider({ + fetchImpl, + baseUrl: 'http://localhost:11434/', + }).summarizeSession(input); expect(note.topics).toEqual(['release']); expect(urls[0]).toBe('http://localhost:11434/api/chat'); }); it('throws on a non-ok response', async () => { const fetchImpl: FetchLike = () => - Promise.resolve({ ok: false, status: 500, statusText: 'Server Error', json: () => Promise.resolve({}) }); - await expect(createOllamaProvider({ fetchImpl }).summarizeSession(input)).rejects.toThrow('Ollama request failed'); + Promise.resolve({ + ok: false, + status: 500, + statusText: 'Server Error', + json: () => Promise.resolve({}), + }); + await expect(createOllamaProvider({ fetchImpl }).summarizeSession(input)).rejects.toThrow( + 'Ollama request failed' + ); }); }); diff --git a/packages/ai-core/src/schemas.test.ts b/packages/ai-core/src/schemas.test.ts index a2f91cfa..f88ba1ca 100644 --- a/packages/ai-core/src/schemas.test.ts +++ b/packages/ai-core/src/schemas.test.ts @@ -6,7 +6,9 @@ describe('sessionNoteSchema', () => { const result = sessionNoteSchema.safeParse({ tldr: 'We fixed the flaky auth test.', decisions: ['Pin the clock in tests'], - actionItems: [{ description: 'Backport the fix', owner: 'Ada', deadline: 'Friday', timestampMs: 120000 }], + actionItems: [ + { description: 'Backport the fix', owner: 'Ada', deadline: 'Friday', timestampMs: 120000 }, + ], topics: ['auth', 'testing'], }); expect(result.success).toBe(true); @@ -16,14 +18,21 @@ describe('sessionNoteSchema', () => { const result = sessionNoteSchema.safeParse({ tldr: 'x', decisions: [], - actionItems: [{ description: 'Do the thing', owner: null, deadline: null, timestampMs: null }], + actionItems: [ + { description: 'Do the thing', owner: null, deadline: null, timestampMs: null }, + ], topics: [], }); expect(result.success).toBe(true); }); it('rejects an empty tldr', () => { - const result = sessionNoteSchema.safeParse({ tldr: '', decisions: [], actionItems: [], topics: [] }); + const result = sessionNoteSchema.safeParse({ + tldr: '', + decisions: [], + actionItems: [], + topics: [], + }); expect(result.success).toBe(false); }); }); @@ -53,7 +62,13 @@ describe('clipCandidateSchema', () => { }); describe('clipCandidatesSchema', () => { - const clip = { startMs: 0, endMs: 1000, title: 't', hookCaption: 'h', platformFit: ['shorts'] as const }; + const clip = { + startMs: 0, + endMs: 1000, + title: 't', + hookCaption: 'h', + platformFit: ['shorts'] as const, + }; it('rejects an empty array', () => { expect(clipCandidatesSchema.safeParse([]).success).toBe(false); diff --git a/packages/ai-core/src/templates.ts b/packages/ai-core/src/templates.ts index 6713cb2e..717e7c2c 100644 --- a/packages/ai-core/src/templates.ts +++ b/packages/ai-core/src/templates.ts @@ -2,7 +2,11 @@ * Built-in summary templates. The PRD ships three at launch; the guidance text * is injected into the summarization system prompt to steer what the notes emphasize. */ -export const SESSION_TEMPLATE_IDS = ['pair-programming', 'support-session', 'user-interview'] as const; +export const SESSION_TEMPLATE_IDS = [ + 'pair-programming', + 'support-session', + 'user-interview', +] as const; export type SessionTemplateId = (typeof SESSION_TEMPLATE_IDS)[number]; export interface SessionTemplateSpec { diff --git a/packages/ai-core/src/transcript.test.ts b/packages/ai-core/src/transcript.test.ts index d6babbd1..3e245857 100644 --- a/packages/ai-core/src/transcript.test.ts +++ b/packages/ai-core/src/transcript.test.ts @@ -15,7 +15,9 @@ describe('formatTimestamp', () => { describe('formatSegment / formatTranscript', () => { it('renders one segment', () => { - expect(formatSegment({ startMs: 65000, endMs: 70000, speaker: 'Host', text: 'hi' })).toBe('[01:05 Host] hi'); + expect(formatSegment({ startMs: 65000, endMs: 70000, speaker: 'Host', text: 'hi' })).toBe( + '[01:05 Host] hi' + ); }); it('joins segments with newlines', () => {