computer: Add bounded multimodal reads

.changeset/computer-multimodal-reads.md

@@ -0,0 +1,5 @@
+---
+"@cloudflare/computer": minor
+---
+
+Add line formatting and byte continuations to `read`, with bounded image and PDF model output.

packages/computer/src/tools/ai.test.ts

@@ -28,6 +28,16 @@ async function executeTool(tool: unknown, input: unknown): Promise<unknown> {
   return output;
 }
 
+async function modelOutput(tool: unknown, input: unknown, output: unknown): Promise<unknown> {
+  const toModelOutput = (
+    tool as {
+      toModelOutput?: (options: { input: unknown; output: unknown }) => unknown;
+    }
+  ).toModelOutput;
+  if (!toModelOutput) throw new Error("tool has no toModelOutput function");
+  return toModelOutput({ input, output });
+}
+
 async function collectTool(tool: unknown, input: unknown): Promise<unknown[]> {
   const execute = (tool as { execute?: (input: unknown, options: typeof toolOptions) => unknown })
     .execute;
@@ -623,8 +633,174 @@ describe("createAITools filesystem tools", () => {
     const tool = createReadTool({ store: memoryStore({ content: "abcdef\n" }), maxBytes: 3 });
 
     await expect(executeTool(tool, { path: "/workspace/file.txt" })).resolves.toEqual({
-      error:
-        "Line 1 exceeds the 3-byte read cap. Increase the cap or read a narrower range with offset/limit.",
+      error: "Line 1 exceeds the 3-byte read cap. Increase the cap or configure lineTruncation.",
+    });
+  });
+
+  it("optionally includes line numbers", async () => {
+    const store = memoryStore({ content: "one\ntwo\n" });
+    const plain = createReadTool({ store });
+    const numbered = createReadTool({ store, includeLineNumbers: true });
+
+    await expect(executeTool(plain, { path: "/workspace/file.txt" })).resolves.toMatchObject({
+      content: "one\ntwo",
+    });
+    await expect(executeTool(numbered, { path: "/workspace/file.txt" })).resolves.toMatchObject({
+      content: "1\tone\n2\ttwo",
+    });
+  });
+
+  it("truncates long lines by characters or UTF-8 bytes", async () => {
+    const store = memoryStore({ content: "a😀bc\n" });
+    const byChars = createReadTool({ store, lineTruncation: { chars: 2 } });
+    const byBytes = createReadTool({ store, lineTruncation: { bytes: 5 } });
+
+    await expect(executeTool(byChars, { path: "/workspace/file.txt" })).resolves.toMatchObject({
+      content: "a😀... (truncated)",
+    });
+    await expect(executeTool(byBytes, { path: "/workspace/file.txt" })).resolves.toMatchObject({
+      content: "a😀... (truncated)",
+    });
+  });
+
+  it("continues from the first unread byte on the next page", async () => {
+    const content = bytes("first\nsecond\nthird\n");
+    const offsets: number[] = [];
+    const store: FileStore = {
+      async stat() {
+        return { size: content.length, mtime: 1 };
+      },
+      async *readChunks(_path, byteOffset = 0, byteLength) {
+        offsets.push(byteOffset);
+        yield content.slice(
+          byteOffset,
+          byteLength === undefined ? undefined : byteOffset + byteLength,
+        );
+      },
+      async readAll() {
+        return content;
+      },
+      async write() {},
+    };
+    const tool = createReadTool({ store });
+    const first = (await executeTool(tool, {
+      path: "/workspace/file.txt",
+      limit: 1,
+    })) as { nextOffset: number; nextByteOffset: number };
+    await executeTool(tool, {
+      path: "/workspace/file.txt",
+      offset: first.nextOffset,
+      byteOffset: first.nextByteOffset,
+      limit: 1,
+    });
+
+    expect(first).toMatchObject({ nextOffset: 2, nextByteOffset: 6 });
+    expect(offsets).toEqual([0, 6]);
+  });
+
+  it("stops pulling chunks as soon as the line cap is complete", async () => {
+    const chunks = [bytes("first\nsecond"), bytes(" line continues"), bytes(" to the end")];
+    const size = chunks.reduce((total, chunk) => total + chunk.byteLength, 0);
+    let chunksRead = 0;
+    const store: FileStore = {
+      async stat() {
+        return { size, mtime: 1 };
+      },
+      async *readChunks() {
+        for (const chunk of chunks) {
+          chunksRead += 1;
+          yield chunk;
+        }
+      },
+      async readAll() {
+        return null;
+      },
+      async write() {},
+    };
+    const tool = createReadTool({ store });
+
+    await expect(
+      executeTool(tool, { path: "/workspace/file.txt", limit: 1 }),
+    ).resolves.toMatchObject({
+      content: "first",
+      truncated: true,
+      nextOffset: 2,
+      nextByteOffset: 6,
+    });
+    expect(chunksRead).toBe(1);
+  });
+
+  it("returns image extensions as file-data model output", async () => {
+    const content = new Uint8Array([0x89, 0x50, 0x4e, 0x47]);
+    const store = memoryStore({ size: content.length });
+    store.readAll = async () => content;
+    const tool = createReadTool({ store });
+    const output = await executeTool(tool, { path: "/workspace/image.png" });
+
+    expect(output).toMatchObject({
+      kind: "image",
+      mediaType: "image/png",
+      sizeBytes: content.length,
+    });
+    await expect(modelOutput(tool, { path: "/workspace/image.png" }, output)).resolves.toEqual({
+      type: "content",
+      value: [
+        { type: "text", text: "Read /workspace/image.png (image/png, 4 bytes)." },
+        {
+          type: "file-data",
+          data: "iVBORw==",
+          mediaType: "image/png",
+          filename: "image.png",
+        },
+      ],
+    });
+  });
+
+  it("sniffs only a bounded prefix for files without a known extension", async () => {
+    const content = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d, ...bytes("body")]);
+    const ranges: Array<{ offset: number; length: number | undefined }> = [];
+    const store = memoryStore({ size: content.length });
+    store.readChunks = async function* (_path, offset = 0, length) {
+      ranges.push({ offset, length });
+      yield content.slice(offset, length === undefined ? undefined : offset + length);
+    };
+    const tool = createReadTool({ store });
+
+    await expect(executeTool(tool, { path: "/workspace/upload" })).resolves.toMatchObject({
+      kind: "file",
+      mediaType: "application/pdf",
+    });
+    expect(ranges).toEqual([{ offset: 0, length: 512 }]);
+  });
+
+  it("rejects oversized inline media before reading the whole file", async () => {
+    let readAll = false;
+    const store = memoryStore({ size: 10 });
+    store.readAll = async () => {
+      readAll = true;
+      return new Uint8Array(10);
+    };
+    const tool = createReadTool({ store, maxModelBytes: 4 });
+    const output = await executeTool(tool, { path: "/workspace/image.png" });
+
+    await expect(modelOutput(tool, { path: "/workspace/image.png" }, output)).resolves.toEqual({
+      type: "error-text",
+      value:
+        "Read /workspace/image.png (image/png, 10 bytes), but it exceeds the 4-byte inline model output limit.",
+    });
+    expect(readAll).toBe(false);
+  });
+
+  it("rechecks the inline media cap after reading a concurrently changed file", async () => {
+    const store = memoryStore({ size: 2 });
+    store.readAll = async () => new Uint8Array(10);
+    const tool = createReadTool({ store, maxModelBytes: 4 });
+    const output = await executeTool(tool, { path: "/workspace/image.png" });
+
+    await expect(modelOutput(tool, { path: "/workspace/image.png" }, output)).resolves.toEqual({
+      type: "error-text",
+      value:
+        "Read /workspace/image.png (image/png, 10 bytes), but it exceeds the 4-byte inline model output limit.",
     });
   });
 });

packages/computer/src/tools/fs/media.ts

@@ -0,0 +1,130 @@
+import type { FileStore } from "./types.js";
+
+export type DetectedMedia =
+  | { kind: "image"; mediaType: string }
+  | { kind: "file"; mediaType: "application/pdf" }
+  | { kind: "binary"; mediaType: string }
+  | { kind: "text"; mediaType: string };
+
+const EXTENSIONS = new Map<string, DetectedMedia>([
+  [".png", { kind: "image", mediaType: "image/png" }],
+  [".jpg", { kind: "image", mediaType: "image/jpeg" }],
+  [".jpeg", { kind: "image", mediaType: "image/jpeg" }],
+  [".gif", { kind: "image", mediaType: "image/gif" }],
+  [".webp", { kind: "image", mediaType: "image/webp" }],
+  [".svg", { kind: "image", mediaType: "image/svg+xml" }],
+  [".pdf", { kind: "file", mediaType: "application/pdf" }],
+]);
+
+const TEXT_EXTENSIONS = new Set([
+  ".c",
+  ".cc",
+  ".cpp",
+  ".css",
+  ".csv",
+  ".go",
+  ".h",
+  ".html",
+  ".java",
+  ".js",
+  ".json",
+  ".jsonc",
+  ".jsx",
+  ".md",
+  ".mjs",
+  ".py",
+  ".rs",
+  ".sh",
+  ".toml",
+  ".ts",
+  ".tsx",
+  ".txt",
+  ".xml",
+  ".yaml",
+  ".yml",
+  ".zig",
+]);
+
+export async function detectMedia(
+  store: FileStore,
+  path: string,
+  sniffBytes: number,
+): Promise<DetectedMedia> {
+  const extension = extensionOf(path);
+  const known = EXTENSIONS.get(extension);
+  if (known !== undefined) return known;
+  if (TEXT_EXTENSIONS.has(extension)) return { kind: "text", mediaType: "text/plain" };
+
+  const prefix = await readPrefix(store, path, sniffBytes);
+  if (startsWith(prefix, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) {
+    return { kind: "image", mediaType: "image/png" };
+  }
+  if (startsWith(prefix, [0xff, 0xd8, 0xff])) {
+    return { kind: "image", mediaType: "image/jpeg" };
+  }
+  if (startsWithAscii(prefix, "GIF87a") || startsWithAscii(prefix, "GIF89a")) {
+    return { kind: "image", mediaType: "image/gif" };
+  }
+  if (startsWithAscii(prefix, "RIFF") && asciiAt(prefix, 8, 12) === "WEBP") {
+    return { kind: "image", mediaType: "image/webp" };
+  }
+  if (startsWithAscii(prefix, "%PDF-")) {
+    return { kind: "file", mediaType: "application/pdf" };
+  }
+  if (looksLikeSvg(prefix)) return { kind: "image", mediaType: "image/svg+xml" };
+  if (looksLikeText(prefix)) return { kind: "text", mediaType: "text/plain" };
+  return { kind: "binary", mediaType: "application/octet-stream" };
+}
+
+async function readPrefix(store: FileStore, path: string, length: number): Promise<Uint8Array> {
+  const parts: Uint8Array[] = [];
+  let total = 0;
+  for await (const chunk of store.readChunks(path, 0, length)) {
+    parts.push(chunk);
+    total += chunk.byteLength;
+  }
+  if (parts.length === 0) return new Uint8Array();
+  if (parts.length === 1) return parts[0];
+  const result = new Uint8Array(total);
+  let offset = 0;
+  for (const part of parts) {
+    result.set(part, offset);
+    offset += part.byteLength;
+  }
+  return result;
+}
+
+function extensionOf(path: string): string {
+  const name = path.slice(path.lastIndexOf("/") + 1).toLowerCase();
+  const dot = name.lastIndexOf(".");
+  return dot <= 0 ? "" : name.slice(dot);
+}
+
+function startsWith(bytes: Uint8Array, prefix: number[]): boolean {
+  return bytes.length >= prefix.length && prefix.every((byte, index) => bytes[index] === byte);
+}
+
+function startsWithAscii(bytes: Uint8Array, prefix: string): boolean {
+  return asciiAt(bytes, 0, prefix.length) === prefix;
+}
+
+function asciiAt(bytes: Uint8Array, start: number, end: number): string {
+  return String.fromCharCode(...bytes.subarray(start, end));
+}
+
+function looksLikeSvg(bytes: Uint8Array): boolean {
+  const prefix = new TextDecoder().decode(bytes).trimStart().toLowerCase();
+  return prefix.startsWith("<svg") || prefix.includes("<svg");
+}
+
+function looksLikeText(bytes: Uint8Array): boolean {
+  if (bytes.length === 0) return true;
+  if (bytes.includes(0)) return false;
+  const text = new TextDecoder().decode(bytes);
+  if (text.length === 0) return true;
+  let replacements = 0;
+  for (const char of text) {
+    if (char === "\uFFFD") replacements += 1;
+  }
+  return replacements / text.length < 0.01;
+}