Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/features/workspaces/extraction/liteparse-projection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ export async function publishLiteParseProjection(
providerMode: "fast",
sourceHash: object.etag,
metadataJson: {
hasExtractableText: projection.hasExtractableText,
markdownLength: projection.manifest.markdownLength,
pageCount: projection.manifest.pageCount,
provisional: true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ export class WorkspaceFileExtractionWorkflow extends WorkflowEntrypoint<
});

return {
hasExtractableText: projection.hasExtractableText,
manifestObjectKey: projection.manifestObjectKey,
markdownLength: projection.manifest.markdownLength,
provider: extraction.provider,
Expand Down Expand Up @@ -121,6 +122,7 @@ export class WorkspaceFileExtractionWorkflow extends WorkflowEntrypoint<
routeReason: extraction.routeReason,
pageCount: extraction.pageCount,
markdownLength: extraction.markdownLength,
hasExtractableText: extraction.hasExtractableText,
};

await kernel.upsertFileProjection({
Expand Down Expand Up @@ -239,6 +241,7 @@ export class WorkspaceFileExtractionWorkflow extends WorkflowEntrypoint<
}

interface StagedPageExtractionResult {
hasExtractableText: boolean;
manifestObjectKey: string;
markdownLength: number;
provider: MarkdownExtractionProviderId;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,78 @@ describe("workspace page projections", () => {
});
});

it("resolves to an empty projection when a PDF has no extractable text", async () => {
const storage = createObjectStorage();
const reference = await writeWorkspacePageProjection({
bucket: storage.bucket,
itemId: "item-1",
pages: [
{ pageNumber: 1, markdown: "" },
{ pageNumber: 2, markdown: "" },
],
provider: "liteparse",
providerMode: "fast",
runId: "run-1",
sourceHash: "etag-1",
tier: "fast",
workspaceId: "workspace-1",
});

expect(reference.hasExtractableText).toBe(false);
expect(reference.manifest.hasExtractableText).toBe(false);
expect(reference.manifest.markdownLength).toBe(0);
expect(reference.manifest.pageCount).toBe(2);

await expect(
readWorkspacePageProjection({
bucket: storage.bucket,
expectedSourceHash: "etag-1",
manifestObjectKey: reference.manifestObjectKey,
pages: "1-2",
}),
).resolves.toEqual({
content: "## Page 1\n\n## Page 2",
pages: { requested: "1-2", returned: [1, 2], total: 2 },
});
});

it("flags projections that contain extractable text", async () => {
const storage = createObjectStorage();
const reference = await writeWorkspacePageProjection({
bucket: storage.bucket,
itemId: "item-1",
pages: [{ pageNumber: 1, markdown: "First" }],
provider: "liteparse",
providerMode: "fast",
runId: "run-1",
sourceHash: "etag-1",
tier: "fast",
workspaceId: "workspace-1",
});

expect(reference.hasExtractableText).toBe(true);
expect(reference.manifest.hasExtractableText).toBe(true);
});

it("rejects extractions that produce no pages at all", async () => {
const storage = createObjectStorage();

await expect(
writeWorkspacePageProjection({
bucket: storage.bucket,
itemId: "item-1",
pages: [],
provider: "liteparse",
providerMode: "fast",
runId: "run-1",
sourceHash: "etag-1",
tier: "fast",
workspaceId: "workspace-1",
}),
).rejects.toThrow("Extraction did not produce any pages.");
expect(storage.values.size).toBe(0);
});

it("removes partial artifacts when publication fails", async () => {
const storage = createObjectStorage();

Expand Down
13 changes: 10 additions & 3 deletions src/features/workspaces/extraction/workspace-page-projection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ const workspacePageProjectionManifestPageSchema = z.object({

const workspacePageProjectionManifestSchema = z.object({
createdAt: z.string(),
hasExtractableText: z.boolean().optional(),
itemId: z.string(),
markdownBytes: z.number().int().nonnegative(),
markdownLength: z.number().int().nonnegative(),
Expand Down Expand Up @@ -92,12 +93,18 @@ export async function writeWorkspacePageProjection(input: {
}

await flushPageWrites(writes);
if (lastPageNumber === 0 || usablePageCount === 0) {
throw new Error("Extraction did not produce usable page Markdown.");
// A page-less extraction means the provider gave us nothing to work with, which is a
// genuine failure. Pages that all trim to empty Markdown, on the other hand, are a valid
// outcome for scanned or image-only PDFs with no text layer: we keep the blank pages and
// record that the file has no extractable text rather than rejecting the extraction.
if (lastPageNumber === 0) {
throw new Error("Extraction did not produce any pages.");
}
const hasExtractableText = usablePageCount > 0;

const manifest: WorkspacePageProjectionManifest = {
createdAt: new Date().toISOString(),
hasExtractableText,
itemId: input.itemId,
markdownBytes,
markdownLength,
Expand All @@ -116,7 +123,7 @@ export async function writeWorkspacePageProjection(input: {
httpMetadata: { contentType: "application/json" },
});

return { manifest, manifestObjectKey };
return { hasExtractableText, manifest, manifestObjectKey };
} catch (error) {
const cleanupErrors: unknown[] = [];
try {
Expand Down
Loading