From 6f5256fb8d5d5a476bd438a7645759f44da136ae Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:29:27 +0000 Subject: [PATCH] fix(workspaces): retry file processor container provisioning The Containers runtime throws "there is no container instance that can be provided to this durable object" when no instance is available to back the WorkspaceFileProcessor durable object (transient provisioning lag or a fully busy pool). Previously this bubbled straight up through preview generation, extraction, and PDF upload validation, failing the operation on the first miss. Retry acquisition (getRandom + startAndWaitForPorts) with exponential backoff, scoped to that specific transient error; all other errors still surface immediately. The retry wraps only provisioning, which runs before the request body stream is consumed, so it never re-reads a consumed stream. Generated-By: PostHog Code Task-Id: 66783ae2-2a28-419c-b710-2d31c9526cc2 --- .../files/workspace-file-processor.test.ts | 87 +++++++++++++++++++ .../files/workspace-file-processor.ts | 49 ++++++++++- 2 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 src/features/workspaces/files/workspace-file-processor.test.ts diff --git a/src/features/workspaces/files/workspace-file-processor.test.ts b/src/features/workspaces/files/workspace-file-processor.test.ts new file mode 100644 index 000000000..fed5b3226 --- /dev/null +++ b/src/features/workspaces/files/workspace-file-processor.test.ts @@ -0,0 +1,87 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const getRandom = vi.hoisted(() => vi.fn()); + +vi.mock("@cloudflare/containers", () => ({ + Container: class {}, + getRandom, +})); + +import { requestWorkspaceFileProcessor } from "#/features/workspaces/files/workspace-file-processor"; + +const startAndWaitForPorts = vi.fn(); +const fetch = vi.fn(); + +describe("requestWorkspaceFileProcessor", () => { + beforeEach(() => { + vi.useFakeTimers(); + getRandom.mockReset(); + startAndWaitForPorts.mockReset(); + fetch.mockReset(); + getRandom.mockResolvedValue({ fetch, startAndWaitForPorts }); + fetch.mockResolvedValue(new Response("ok")); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("retries provisioning when no container instance is available yet", async () => { + startAndWaitForPorts + .mockRejectedValueOnce( + new Error("there is no container instance that can be provided to this durable object"), + ) + .mockResolvedValueOnce(undefined); + + const promise = requestWorkspaceFileProcessor(env(), input()); + await vi.runAllTimersAsync(); + const response = await promise; + + expect(await response.text()).toBe("ok"); + expect(startAndWaitForPorts).toHaveBeenCalledTimes(2); + expect(fetch).toHaveBeenCalledOnce(); + }); + + it("does not retry errors unrelated to provisioning", async () => { + startAndWaitForPorts.mockRejectedValue(new Error("port ready timeout")); + + await expect(requestWorkspaceFileProcessor(env(), input())).rejects.toThrow( + "port ready timeout", + ); + expect(startAndWaitForPorts).toHaveBeenCalledOnce(); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("gives up after exhausting provisioning retries", async () => { + startAndWaitForPorts.mockRejectedValue( + new Error("there is no container instance that can be provided to this durable object"), + ); + + const promise = requestWorkspaceFileProcessor(env(), input()); + const settled = expect(promise).rejects.toThrow("no container instance"); + await vi.runAllTimersAsync(); + await settled; + + expect(startAndWaitForPorts).toHaveBeenCalledTimes(4); + expect(fetch).not.toHaveBeenCalled(); + }); +}); + +function env(): Cloudflare.Env { + return { WORKSPACE_FILE_PROCESSOR: {} } as unknown as Cloudflare.Env; +} + +function input() { + const body = new Response(new Uint8Array([1, 2, 3]).buffer).body; + + if (!body) { + throw new Error("Test stream was not created."); + } + + return { + body, + contentType: "application/pdf", + path: "/preview/pdf" as const, + sizeBytes: 3, + }; +} diff --git a/src/features/workspaces/files/workspace-file-processor.ts b/src/features/workspaces/files/workspace-file-processor.ts index f738a1835..67d2690d6 100644 --- a/src/features/workspaces/files/workspace-file-processor.ts +++ b/src/features/workspaces/files/workspace-file-processor.ts @@ -9,6 +9,13 @@ const processorRequestTimeoutMs = { "/validate/pdf": 2 * 60_000, } as const; +// The Containers runtime throws when no instance is available to back the durable +// object (e.g. provisioning lag right after a deploy, or every instance in the pool +// busy). This is transient, so we retry acquisition with backoff instead of failing +// the caller on the first miss. +const provisioningRetryAttempts = 4; +const provisioningRetryBaseDelayMs = 1_000; + export class WorkspaceFileProcessor extends Container { defaultPort = workspaceFileProcessorPort; requiredPorts = [workspaceFileProcessorPort]; @@ -16,6 +23,43 @@ export class WorkspaceFileProcessor extends Container { enableInternet = false; } +function isContainerProvisioningError(error: unknown): boolean { + return error instanceof Error && error.message.includes("no container instance"); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function acquireWorkspaceFileProcessor(env: Cloudflare.Env) { + let lastError: unknown; + + for (let attempt = 0; attempt < provisioningRetryAttempts; attempt++) { + try { + const processor = await getRandom( + env.WORKSPACE_FILE_PROCESSOR, + workspaceFileProcessorPoolSize, + ); + await processor.startAndWaitForPorts({ + cancellationOptions: { portReadyTimeoutMS: 60_000 }, + }); + return processor; + } catch (error) { + if (!isContainerProvisioningError(error)) { + throw error; + } + + lastError = error; + + if (attempt < provisioningRetryAttempts - 1) { + await sleep(provisioningRetryBaseDelayMs * 2 ** attempt); + } + } + } + + throw lastError; +} + export async function requestWorkspaceFileProcessor( env: Cloudflare.Env, input: { @@ -26,10 +70,7 @@ export async function requestWorkspaceFileProcessor( sizeBytes: number; }, ) { - const processor = await getRandom(env.WORKSPACE_FILE_PROCESSOR, workspaceFileProcessorPoolSize); - await processor.startAndWaitForPorts({ - cancellationOptions: { portReadyTimeoutMS: 60_000 }, - }); + const processor = await acquireWorkspaceFileProcessor(env); const headers = new Headers({ "content-type": input.contentType,