From a5f656c2376e6d24a593ceeeb882d4fef0fc4d03 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 15:40:47 +0000 Subject: [PATCH] fix(workspaces): resolve text-less PDFs to an empty projection Extraction previously threw "Extraction did not produce usable page Markdown." whenever every page came back empty, which happens naturally for scanned or image-only PDFs with no text layer (LiteParse trims each page's Markdown, so whitespace-only pages collapse to empty strings). Distinguish "the provider gave us no pages at all" (still a genuine failure) from "the PDF has pages but no extractable text" (a valid outcome). The latter now resolves into a valid projection with blank pages and a `hasExtractableText: false` marker recorded on the manifest and file projection metadata, so the file is marked done-with-no-text instead of failed. Generated-By: PostHog Code Task-Id: 97353e21-4b24-4afc-9a1f-2846a17dda22 --- .../extraction/liteparse-projection.ts | 1 + .../workspace-file-extraction-workflow.ts | 3 + .../workspace-page-projection.test.ts | 72 +++++++++++++++++++ .../extraction/workspace-page-projection.ts | 13 +++- 4 files changed, 86 insertions(+), 3 deletions(-) diff --git a/src/features/workspaces/extraction/liteparse-projection.ts b/src/features/workspaces/extraction/liteparse-projection.ts index 4766e7877..a17dd481d 100644 --- a/src/features/workspaces/extraction/liteparse-projection.ts +++ b/src/features/workspaces/extraction/liteparse-projection.ts @@ -61,6 +61,7 @@ export async function publishLiteParseProjection( providerMode: "fast", sourceHash: object.etag, metadataJson: { + hasExtractableText: projection.hasExtractableText, markdownLength: projection.manifest.markdownLength, pageCount: projection.manifest.pageCount, provisional: true, diff --git a/src/features/workspaces/extraction/workspace-file-extraction-workflow.ts b/src/features/workspaces/extraction/workspace-file-extraction-workflow.ts index 446226008..c8d6229bc 100644 --- a/src/features/workspaces/extraction/workspace-file-extraction-workflow.ts +++ b/src/features/workspaces/extraction/workspace-file-extraction-workflow.ts @@ -92,6 +92,7 @@ export class WorkspaceFileExtractionWorkflow extends WorkflowEntrypoint< }); return { + hasExtractableText: projection.hasExtractableText, manifestObjectKey: projection.manifestObjectKey, markdownLength: projection.manifest.markdownLength, provider: extraction.provider, @@ -121,6 +122,7 @@ export class WorkspaceFileExtractionWorkflow extends WorkflowEntrypoint< routeReason: extraction.routeReason, pageCount: extraction.pageCount, markdownLength: extraction.markdownLength, + hasExtractableText: extraction.hasExtractableText, }; await kernel.upsertFileProjection({ @@ -239,6 +241,7 @@ export class WorkspaceFileExtractionWorkflow extends WorkflowEntrypoint< } interface StagedPageExtractionResult { + hasExtractableText: boolean; manifestObjectKey: string; markdownLength: number; provider: MarkdownExtractionProviderId; diff --git a/src/features/workspaces/extraction/workspace-page-projection.test.ts b/src/features/workspaces/extraction/workspace-page-projection.test.ts index 61139a0fd..a244224a8 100644 --- a/src/features/workspaces/extraction/workspace-page-projection.test.ts +++ b/src/features/workspaces/extraction/workspace-page-projection.test.ts @@ -217,6 +217,78 @@ describe("workspace page projections", () => { }); }); + it("resolves to an empty projection when a PDF has no extractable text", async () => { + const storage = createObjectStorage(); + const reference = await writeWorkspacePageProjection({ + bucket: storage.bucket, + itemId: "item-1", + pages: [ + { pageNumber: 1, markdown: "" }, + { pageNumber: 2, markdown: "" }, + ], + provider: "liteparse", + providerMode: "fast", + runId: "run-1", + sourceHash: "etag-1", + tier: "fast", + workspaceId: "workspace-1", + }); + + expect(reference.hasExtractableText).toBe(false); + expect(reference.manifest.hasExtractableText).toBe(false); + expect(reference.manifest.markdownLength).toBe(0); + expect(reference.manifest.pageCount).toBe(2); + + await expect( + readWorkspacePageProjection({ + bucket: storage.bucket, + expectedSourceHash: "etag-1", + manifestObjectKey: reference.manifestObjectKey, + pages: "1-2", + }), + ).resolves.toEqual({ + content: "## Page 1\n\n## Page 2", + pages: { requested: "1-2", returned: [1, 2], total: 2 }, + }); + }); + + it("flags projections that contain extractable text", async () => { + const storage = createObjectStorage(); + const reference = await writeWorkspacePageProjection({ + bucket: storage.bucket, + itemId: "item-1", + pages: [{ pageNumber: 1, markdown: "First" }], + provider: "liteparse", + providerMode: "fast", + runId: "run-1", + sourceHash: "etag-1", + tier: "fast", + workspaceId: "workspace-1", + }); + + expect(reference.hasExtractableText).toBe(true); + expect(reference.manifest.hasExtractableText).toBe(true); + }); + + it("rejects extractions that produce no pages at all", async () => { + const storage = createObjectStorage(); + + await expect( + writeWorkspacePageProjection({ + bucket: storage.bucket, + itemId: "item-1", + pages: [], + provider: "liteparse", + providerMode: "fast", + runId: "run-1", + sourceHash: "etag-1", + tier: "fast", + workspaceId: "workspace-1", + }), + ).rejects.toThrow("Extraction did not produce any pages."); + expect(storage.values.size).toBe(0); + }); + it("removes partial artifacts when publication fails", async () => { const storage = createObjectStorage(); diff --git a/src/features/workspaces/extraction/workspace-page-projection.ts b/src/features/workspaces/extraction/workspace-page-projection.ts index 72136fb09..3d0623960 100644 --- a/src/features/workspaces/extraction/workspace-page-projection.ts +++ b/src/features/workspaces/extraction/workspace-page-projection.ts @@ -23,6 +23,7 @@ const workspacePageProjectionManifestPageSchema = z.object({ const workspacePageProjectionManifestSchema = z.object({ createdAt: z.string(), + hasExtractableText: z.boolean().optional(), itemId: z.string(), markdownBytes: z.number().int().nonnegative(), markdownLength: z.number().int().nonnegative(), @@ -92,12 +93,18 @@ export async function writeWorkspacePageProjection(input: { } await flushPageWrites(writes); - if (lastPageNumber === 0 || usablePageCount === 0) { - throw new Error("Extraction did not produce usable page Markdown."); + // A page-less extraction means the provider gave us nothing to work with, which is a + // genuine failure. Pages that all trim to empty Markdown, on the other hand, are a valid + // outcome for scanned or image-only PDFs with no text layer: we keep the blank pages and + // record that the file has no extractable text rather than rejecting the extraction. + if (lastPageNumber === 0) { + throw new Error("Extraction did not produce any pages."); } + const hasExtractableText = usablePageCount > 0; const manifest: WorkspacePageProjectionManifest = { createdAt: new Date().toISOString(), + hasExtractableText, itemId: input.itemId, markdownBytes, markdownLength, @@ -116,7 +123,7 @@ export async function writeWorkspacePageProjection(input: { httpMetadata: { contentType: "application/json" }, }); - return { manifest, manifestObjectKey }; + return { hasExtractableText, manifest, manifestObjectKey }; } catch (error) { const cleanupErrors: unknown[] = []; try {