diff --git a/src/features/workspaces/files/workspace-file-processor.test.ts b/src/features/workspaces/files/workspace-file-processor.test.ts new file mode 100644 index 000000000..fed5b3226 --- /dev/null +++ b/src/features/workspaces/files/workspace-file-processor.test.ts @@ -0,0 +1,87 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const getRandom = vi.hoisted(() => vi.fn()); + +vi.mock("@cloudflare/containers", () => ({ + Container: class {}, + getRandom, +})); + +import { requestWorkspaceFileProcessor } from "#/features/workspaces/files/workspace-file-processor"; + +const startAndWaitForPorts = vi.fn(); +const fetch = vi.fn(); + +describe("requestWorkspaceFileProcessor", () => { + beforeEach(() => { + vi.useFakeTimers(); + getRandom.mockReset(); + startAndWaitForPorts.mockReset(); + fetch.mockReset(); + getRandom.mockResolvedValue({ fetch, startAndWaitForPorts }); + fetch.mockResolvedValue(new Response("ok")); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("retries provisioning when no container instance is available yet", async () => { + startAndWaitForPorts + .mockRejectedValueOnce( + new Error("there is no container instance that can be provided to this durable object"), + ) + .mockResolvedValueOnce(undefined); + + const promise = requestWorkspaceFileProcessor(env(), input()); + await vi.runAllTimersAsync(); + const response = await promise; + + expect(await response.text()).toBe("ok"); + expect(startAndWaitForPorts).toHaveBeenCalledTimes(2); + expect(fetch).toHaveBeenCalledOnce(); + }); + + it("does not retry errors unrelated to provisioning", async () => { + startAndWaitForPorts.mockRejectedValue(new Error("port ready timeout")); + + await expect(requestWorkspaceFileProcessor(env(), input())).rejects.toThrow( + "port ready timeout", + ); + expect(startAndWaitForPorts).toHaveBeenCalledOnce(); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("gives up after exhausting provisioning retries", async () => { + startAndWaitForPorts.mockRejectedValue( + new Error("there is no container instance that can be provided to this durable object"), + ); + + const promise = requestWorkspaceFileProcessor(env(), input()); + const settled = expect(promise).rejects.toThrow("no container instance"); + await vi.runAllTimersAsync(); + await settled; + + expect(startAndWaitForPorts).toHaveBeenCalledTimes(4); + expect(fetch).not.toHaveBeenCalled(); + }); +}); + +function env(): Cloudflare.Env { + return { WORKSPACE_FILE_PROCESSOR: {} } as unknown as Cloudflare.Env; +} + +function input() { + const body = new Response(new Uint8Array([1, 2, 3]).buffer).body; + + if (!body) { + throw new Error("Test stream was not created."); + } + + return { + body, + contentType: "application/pdf", + path: "/preview/pdf" as const, + sizeBytes: 3, + }; +} diff --git a/src/features/workspaces/files/workspace-file-processor.ts b/src/features/workspaces/files/workspace-file-processor.ts index f738a1835..67d2690d6 100644 --- a/src/features/workspaces/files/workspace-file-processor.ts +++ b/src/features/workspaces/files/workspace-file-processor.ts @@ -9,6 +9,13 @@ const processorRequestTimeoutMs = { "/validate/pdf": 2 * 60_000, } as const; +// The Containers runtime throws when no instance is available to back the durable +// object (e.g. provisioning lag right after a deploy, or every instance in the pool +// busy). This is transient, so we retry acquisition with backoff instead of failing +// the caller on the first miss. +const provisioningRetryAttempts = 4; +const provisioningRetryBaseDelayMs = 1_000; + export class WorkspaceFileProcessor extends Container { defaultPort = workspaceFileProcessorPort; requiredPorts = [workspaceFileProcessorPort]; @@ -16,6 +23,43 @@ export class WorkspaceFileProcessor extends Container { enableInternet = false; } +function isContainerProvisioningError(error: unknown): boolean { + return error instanceof Error && error.message.includes("no container instance"); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function acquireWorkspaceFileProcessor(env: Cloudflare.Env) { + let lastError: unknown; + + for (let attempt = 0; attempt < provisioningRetryAttempts; attempt++) { + try { + const processor = await getRandom( + env.WORKSPACE_FILE_PROCESSOR, + workspaceFileProcessorPoolSize, + ); + await processor.startAndWaitForPorts({ + cancellationOptions: { portReadyTimeoutMS: 60_000 }, + }); + return processor; + } catch (error) { + if (!isContainerProvisioningError(error)) { + throw error; + } + + lastError = error; + + if (attempt < provisioningRetryAttempts - 1) { + await sleep(provisioningRetryBaseDelayMs * 2 ** attempt); + } + } + } + + throw lastError; +} + export async function requestWorkspaceFileProcessor( env: Cloudflare.Env, input: { @@ -26,10 +70,7 @@ export async function requestWorkspaceFileProcessor( sizeBytes: number; }, ) { - const processor = await getRandom(env.WORKSPACE_FILE_PROCESSOR, workspaceFileProcessorPoolSize); - await processor.startAndWaitForPorts({ - cancellationOptions: { portReadyTimeoutMS: 60_000 }, - }); + const processor = await acquireWorkspaceFileProcessor(env); const headers = new Headers({ "content-type": input.contentType,