fix(workspaces): retry file processor container provisioning

src/features/workspaces/files/workspace-file-processor.test.ts

@@ -0,0 +1,87 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const getRandom = vi.hoisted(() => vi.fn());
+
+vi.mock("@cloudflare/containers", () => ({
+	Container: class {},
+	getRandom,
+}));
+
+import { requestWorkspaceFileProcessor } from "#/features/workspaces/files/workspace-file-processor";
+
+const startAndWaitForPorts = vi.fn();
+const fetch = vi.fn();
+
+describe("requestWorkspaceFileProcessor", () => {
+	beforeEach(() => {
+		vi.useFakeTimers();
+		getRandom.mockReset();
+		startAndWaitForPorts.mockReset();
+		fetch.mockReset();
+		getRandom.mockResolvedValue({ fetch, startAndWaitForPorts });
+		fetch.mockResolvedValue(new Response("ok"));
+	});
+
+	afterEach(() => {
+		vi.useRealTimers();
+	});
+
+	it("retries provisioning when no container instance is available yet", async () => {
+		startAndWaitForPorts
+			.mockRejectedValueOnce(
+				new Error("there is no container instance that can be provided to this durable object"),
+			)
+			.mockResolvedValueOnce(undefined);
+
+		const promise = requestWorkspaceFileProcessor(env(), input());
+		await vi.runAllTimersAsync();
+		const response = await promise;
+
+		expect(await response.text()).toBe("ok");
+		expect(startAndWaitForPorts).toHaveBeenCalledTimes(2);
+		expect(fetch).toHaveBeenCalledOnce();
+	});
+
+	it("does not retry errors unrelated to provisioning", async () => {
+		startAndWaitForPorts.mockRejectedValue(new Error("port ready timeout"));
+
+		await expect(requestWorkspaceFileProcessor(env(), input())).rejects.toThrow(
+			"port ready timeout",
+		);
+		expect(startAndWaitForPorts).toHaveBeenCalledOnce();
+		expect(fetch).not.toHaveBeenCalled();
+	});
+
+	it("gives up after exhausting provisioning retries", async () => {
+		startAndWaitForPorts.mockRejectedValue(
+			new Error("there is no container instance that can be provided to this durable object"),
+		);
+
+		const promise = requestWorkspaceFileProcessor(env(), input());
+		const settled = expect(promise).rejects.toThrow("no container instance");
+		await vi.runAllTimersAsync();
+		await settled;
+
+		expect(startAndWaitForPorts).toHaveBeenCalledTimes(4);
+		expect(fetch).not.toHaveBeenCalled();
+	});
+});
+
+function env(): Cloudflare.Env {
+	return { WORKSPACE_FILE_PROCESSOR: {} } as unknown as Cloudflare.Env;
+}
+
+function input() {
+	const body = new Response(new Uint8Array([1, 2, 3]).buffer).body;
+
+	if (!body) {
+		throw new Error("Test stream was not created.");
+	}
+
+	return {
+		body,
+		contentType: "application/pdf",
+		path: "/preview/pdf" as const,
+		sizeBytes: 3,
+	};
+}

src/features/workspaces/files/workspace-file-processor.ts

@@ -9,13 +9,57 @@ const processorRequestTimeoutMs = {
 	"/validate/pdf": 2 * 60_000,
 } as const;
 
+// The Containers runtime throws when no instance is available to back the durable
+// object (e.g. provisioning lag right after a deploy, or every instance in the pool
+// busy). This is transient, so we retry acquisition with backoff instead of failing
+// the caller on the first miss.
+const provisioningRetryAttempts = 4;
+const provisioningRetryBaseDelayMs = 1_000;
+
 export class WorkspaceFileProcessor extends Container {
 	defaultPort = workspaceFileProcessorPort;
 	requiredPorts = [workspaceFileProcessorPort];
 	sleepAfter = "5m";
 	enableInternet = false;
 }
 
+function isContainerProvisioningError(error: unknown): boolean {
+	return error instanceof Error && error.message.includes("no container instance");
+}
+
+function sleep(ms: number): Promise<void> {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function acquireWorkspaceFileProcessor(env: Cloudflare.Env) {
+	let lastError: unknown;
+
+	for (let attempt = 0; attempt < provisioningRetryAttempts; attempt++) {
+		try {
+			const processor = await getRandom(
+				env.WORKSPACE_FILE_PROCESSOR,
+				workspaceFileProcessorPoolSize,
+			);
+			await processor.startAndWaitForPorts({
+				cancellationOptions: { portReadyTimeoutMS: 60_000 },
+			});
+			return processor;
+		} catch (error) {
+			if (!isContainerProvisioningError(error)) {
+				throw error;
+			}
+
+			lastError = error;
+
+			if (attempt < provisioningRetryAttempts - 1) {
+				await sleep(provisioningRetryBaseDelayMs * 2 ** attempt);
+			}
+		}
+	}
+
+	throw lastError;
+}
+
 export async function requestWorkspaceFileProcessor(
 	env: Cloudflare.Env,
 	input: {
@@ -26,10 +70,7 @@ export async function requestWorkspaceFileProcessor(
 		sizeBytes: number;
 	},
 ) {
-	const processor = await getRandom(env.WORKSPACE_FILE_PROCESSOR, workspaceFileProcessorPoolSize);
-	await processor.startAndWaitForPorts({
-		cancellationOptions: { portReadyTimeoutMS: 60_000 },
-	});
+	const processor = await acquireWorkspaceFileProcessor(env);
 
 	const headers = new Headers({
 		"content-type": input.contentType,