diff --git a/src/Tests/PdfNormalizerTests.cs b/src/Tests/PdfNormalizerTests.cs new file mode 100644 index 0000000..c57ed74 --- /dev/null +++ b/src/Tests/PdfNormalizerTests.cs @@ -0,0 +1,91 @@ +using System.Text; +using Docnet.Core; + +[TestFixture] +public class PdfNormalizerTests +{ + [Test] + public void NeutralizesVolatileValues() + { + var input = + "/ID [ <1122334455667788>] " + + "/CreationDate(D:20240115093000+05'30') " + + "/ModDate(D:20240115093000Z) " + + "2024-01-15T09:30:00+05:30" + + "2024-01-15T09:30:00Z" + + "2024-01-15T09:30:00Z" + + "uuid:0f7b2c9a-1234-5678-9abc-def012345678" + + "xmp.iid:1a2b3c4d"; + var expected = + "/ID [<0000000000000000> <0000000000000000>] " + + "/CreationDate(D:00000000000000+00'00') " + + "/ModDate(D:00000000000000Z) " + + "0000-00-00T00:00:00+00:00" + + "0000-00-00T00:00:00Z" + + "0000-00-00T00:00:00Z" + + $"{new string('0', 41)}" + + $"{new string('0', 16)}"; + Assert.That(Normalize(input), Is.EqualTo(expected)); + } + + [Test] + public void CollapsesDifferingValuesToTheSameOutput() + { + // The same producer emits a stable structure across runs, so two documents differing only + // in the volatile digits/hex normalize to identical bytes. + var a = "/ID [] /CreationDate(D:20240115093000+05'30')"; + var b = "/ID [<99887766>] /CreationDate(D:19991231235959+11'45')"; + Assert.That(a, Is.Not.EqualTo(b)); + Assert.That(Normalize(a), Is.EqualTo(Normalize(b))); + } + + [Test] + public void LeavesLookalikeKeysUntouched() + { + // /IDTree is a name-tree key (not the file identifier), /ModDateStamp is a different name, + // and a self-closing date element has no content: none should be altered. + var input = "/IDTree [1 2] /ModDateStamp(20240101) 2024"; + Assert.That(Normalize(input), Is.EqualTo(input)); + } + + [Test] + public void NormalizedDocumentStillLoads() + { + var data = File.ReadAllBytes("sample.pdf"); + PdfNormalizer.Normalize(data); + + using var reader = DocLib.Instance.GetDocReader(data, new(scalingFactor: 2)); + Assert.That(reader.GetPageCount(), Is.EqualTo(2)); + } + + [Test] + public void IsIdempotent() + { + // A second pass has nothing left to change: normalizing already-normalized bytes is a no-op. + var once = File.ReadAllBytes("sample.pdf"); + PdfNormalizer.Normalize(once); + var twice = (byte[])once.Clone(); + PdfNormalizer.Normalize(twice); + Assert.That(twice, Is.EqualTo(once)); + } + + [Test] + public void NormalizedSinglePageSplitStillLoads() + { + // A page subset is re-serialized by pdfium (reintroducing volatile fields) then normalized; + // it must remain a valid one-page document. + var data = File.ReadAllBytes("sample.pdf"); + var split = DocLib.Instance.Split(data, 1, 1); + PdfNormalizer.Normalize(split); + + using var reader = DocLib.Instance.GetDocReader(split, new(scalingFactor: 2)); + Assert.That(reader.GetPageCount(), Is.EqualTo(1)); + } + + static string Normalize(string value) + { + var bytes = Encoding.Latin1.GetBytes(value); + PdfNormalizer.Normalize(bytes); + return Encoding.Latin1.GetString(bytes); + } +} diff --git a/src/Tests/PdfSnapshotTests.cs b/src/Tests/PdfSnapshotTests.cs new file mode 100644 index 0000000..ae3d5a0 --- /dev/null +++ b/src/Tests/PdfSnapshotTests.cs @@ -0,0 +1,21 @@ +using System.Runtime.CompilerServices; +using Docnet.Core; + +[TestFixture] +public class PdfSnapshotTests +{ + // The SinglePage snapshots are subset to only the rendered page, so the accepted pdf must load + // as a one-page document. A readable guard alongside the opaque binary verified files. + [TestCase("Samples.VerifyFirstPage#pdf.verified.pdf", 1)] + [TestCase("Samples.VerifySecondPage#pdf.verified.pdf", 1)] + [TestCase("Samples.VerifyPdf#pdf.verified.pdf", 2)] + public void SnapshotHasExpectedPageCount(string file, int expectedPages) + { + var data = File.ReadAllBytes(Path.Combine(SourceDirectory(), file)); + using var reader = DocLib.Instance.GetDocReader(data, new(scalingFactor: 2)); + Assert.That(reader.GetPageCount(), Is.EqualTo(expectedPages)); + } + + static string SourceDirectory([CallerFilePath] string path = "") => + Path.GetDirectoryName(path)!; +} diff --git a/src/Tests/Samples.VerifyFirstPage#pdf.verified.pdf b/src/Tests/Samples.VerifyFirstPage#pdf.verified.pdf new file mode 100644 index 0000000..ee30c39 Binary files /dev/null and b/src/Tests/Samples.VerifyFirstPage#pdf.verified.pdf differ diff --git a/src/Tests/Samples.VerifyFirstPage.verified.txt b/src/Tests/Samples.VerifyFirstPage.verified.txt new file mode 100644 index 0000000..ac61bc8 --- /dev/null +++ b/src/Tests/Samples.VerifyFirstPage.verified.txt @@ -0,0 +1,19 @@ +{ + Version: 1.4, + PageCount: 2, + Pages: [ + { + Text: + A Simple PDF File + This is a small demonstration .pdf file - + just for use in the Virtual Mechanics tutorials. More text. And more + text. And more text. And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. And more text. Boring, zzzzz. And more text. And more text. And + more text. And more text. And more text. And more text. And more text. + And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. And more text. And more text. Even more. Continued on page 2 ... + } + ] +} \ No newline at end of file diff --git a/src/Tests/Samples.VerifyPageDimensions#pdf.verified.pdf b/src/Tests/Samples.VerifyPageDimensions#pdf.verified.pdf new file mode 100644 index 0000000..acb31ef Binary files /dev/null and b/src/Tests/Samples.VerifyPageDimensions#pdf.verified.pdf differ diff --git a/src/Tests/Samples.VerifyPageDimensions.verified.txt b/src/Tests/Samples.VerifyPageDimensions.verified.txt new file mode 100644 index 0000000..a3766c3 --- /dev/null +++ b/src/Tests/Samples.VerifyPageDimensions.verified.txt @@ -0,0 +1,29 @@ +{ + Version: 1.4, + PageCount: 2, + Pages: [ + { + Text: + A Simple PDF File + This is a small demonstration .pdf file - + just for use in the Virtual Mechanics tutorials. More text. And more + text. And more text. And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. And more text. Boring, zzzzz. And more text. And more text. And + more text. And more text. And more text. And more text. And more text. + And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. And more text. And more text. Even more. Continued on page 2 ... + }, + { + Index: 1, + Text: + Simple PDF File 2 + ...continued from page 1. Yet more text. And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. Oh, how boring typing this stuff. But not as boring as watching + paint dry. And more text. And more text. And more text. And more text. + Boring. More, a little more text. The end, and just as well. + } + ] +} \ No newline at end of file diff --git a/src/Tests/Samples.VerifyPdf#pdf.verified.pdf b/src/Tests/Samples.VerifyPdf#pdf.verified.pdf new file mode 100644 index 0000000..acb31ef Binary files /dev/null and b/src/Tests/Samples.VerifyPdf#pdf.verified.pdf differ diff --git a/src/Tests/Samples.VerifyPdf.verified.txt b/src/Tests/Samples.VerifyPdf.verified.txt new file mode 100644 index 0000000..a3766c3 --- /dev/null +++ b/src/Tests/Samples.VerifyPdf.verified.txt @@ -0,0 +1,29 @@ +{ + Version: 1.4, + PageCount: 2, + Pages: [ + { + Text: + A Simple PDF File + This is a small demonstration .pdf file - + just for use in the Virtual Mechanics tutorials. More text. And more + text. And more text. And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. And more text. Boring, zzzzz. And more text. And more text. And + more text. And more text. And more text. And more text. And more text. + And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. And more text. And more text. Even more. Continued on page 2 ... + }, + { + Index: 1, + Text: + Simple PDF File 2 + ...continued from page 1. Yet more text. And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. Oh, how boring typing this stuff. But not as boring as watching + paint dry. And more text. And more text. And more text. And more text. + Boring. More, a little more text. The end, and just as well. + } + ] +} \ No newline at end of file diff --git a/src/Tests/Samples.VerifyPdfStream#pdf.verified.pdf b/src/Tests/Samples.VerifyPdfStream#pdf.verified.pdf new file mode 100644 index 0000000..acb31ef Binary files /dev/null and b/src/Tests/Samples.VerifyPdfStream#pdf.verified.pdf differ diff --git a/src/Tests/Samples.VerifyPdfStream.verified.txt b/src/Tests/Samples.VerifyPdfStream.verified.txt new file mode 100644 index 0000000..a3766c3 --- /dev/null +++ b/src/Tests/Samples.VerifyPdfStream.verified.txt @@ -0,0 +1,29 @@ +{ + Version: 1.4, + PageCount: 2, + Pages: [ + { + Text: + A Simple PDF File + This is a small demonstration .pdf file - + just for use in the Virtual Mechanics tutorials. More text. And more + text. And more text. And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. And more text. Boring, zzzzz. And more text. And more text. And + more text. And more text. And more text. And more text. And more text. + And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. And more text. And more text. Even more. Continued on page 2 ... + }, + { + Index: 1, + Text: + Simple PDF File 2 + ...continued from page 1. Yet more text. And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. Oh, how boring typing this stuff. But not as boring as watching + paint dry. And more text. And more text. And more text. And more text. + Boring. More, a little more text. The end, and just as well. + } + ] +} \ No newline at end of file diff --git a/src/Tests/Samples.VerifyPreserveTransparency#pdf.verified.pdf b/src/Tests/Samples.VerifyPreserveTransparency#pdf.verified.pdf new file mode 100644 index 0000000..acb31ef Binary files /dev/null and b/src/Tests/Samples.VerifyPreserveTransparency#pdf.verified.pdf differ diff --git a/src/Tests/Samples.VerifyPreserveTransparency.verified.txt b/src/Tests/Samples.VerifyPreserveTransparency.verified.txt new file mode 100644 index 0000000..a3766c3 --- /dev/null +++ b/src/Tests/Samples.VerifyPreserveTransparency.verified.txt @@ -0,0 +1,29 @@ +{ + Version: 1.4, + PageCount: 2, + Pages: [ + { + Text: + A Simple PDF File + This is a small demonstration .pdf file - + just for use in the Virtual Mechanics tutorials. More text. And more + text. And more text. And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. And more text. Boring, zzzzz. And more text. And more text. And + more text. And more text. And more text. And more text. And more text. + And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. And more text. And more text. Even more. Continued on page 2 ... + }, + { + Index: 1, + Text: + Simple PDF File 2 + ...continued from page 1. Yet more text. And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. Oh, how boring typing this stuff. But not as boring as watching + paint dry. And more text. And more text. And more text. And more text. + Boring. More, a little more text. The end, and just as well. + } + ] +} \ No newline at end of file diff --git a/src/Tests/Samples.VerifySecondPage#pdf.verified.pdf b/src/Tests/Samples.VerifySecondPage#pdf.verified.pdf new file mode 100644 index 0000000..1591004 Binary files /dev/null and b/src/Tests/Samples.VerifySecondPage#pdf.verified.pdf differ diff --git a/src/Tests/Samples.VerifySecondPage.verified.txt b/src/Tests/Samples.VerifySecondPage.verified.txt new file mode 100644 index 0000000..485ed0a --- /dev/null +++ b/src/Tests/Samples.VerifySecondPage.verified.txt @@ -0,0 +1,16 @@ +{ + Version: 1.4, + PageCount: 2, + Pages: [ + { + Index: 1, + Text: + Simple PDF File 2 + ...continued from page 1. Yet more text. And more text. And more text. + And more text. And more text. And more text. And more text. And more + text. Oh, how boring typing this stuff. But not as boring as watching + paint dry. And more text. And more text. And more text. And more text. + Boring. More, a little more text. The end, and just as well. + } + ] +} \ No newline at end of file diff --git a/src/Verify.DocNet/PageInfo.cs b/src/Verify.DocNet/PageInfo.cs new file mode 100644 index 0000000..eeceeaf --- /dev/null +++ b/src/Verify.DocNet/PageInfo.cs @@ -0,0 +1,8 @@ +class PageInfo +{ + /// Zero based index of the page within the source document. + public int Index { get; init; } + + /// Text extracted from the page, or null when the page has no text. + public string? Text { get; init; } +} diff --git a/src/Verify.DocNet/PdfInfo.cs b/src/Verify.DocNet/PdfInfo.cs new file mode 100644 index 0000000..4b6ac9f --- /dev/null +++ b/src/Verify.DocNet/PdfInfo.cs @@ -0,0 +1,13 @@ +class PdfInfo +{ + public string Version { get; init; } = ""; + + /// + /// Total number of pages in the source document. This is the full count regardless of any + /// PagesToInclude/SinglePage filter; the pages actually rendered are listed in + /// , each carrying its within the document. + /// + public int PageCount { get; init; } + + public IReadOnlyList Pages { get; init; } = []; +} diff --git a/src/Verify.DocNet/PdfNormalizer.cs b/src/Verify.DocNet/PdfNormalizer.cs new file mode 100644 index 0000000..d85d37c --- /dev/null +++ b/src/Verify.DocNet/PdfNormalizer.cs @@ -0,0 +1,274 @@ +/// +/// Neutralizes the non-deterministic fields of a PDF (the trailer /ID, the document +/// information /CreationDate and /ModDate, and the equivalent XMP metadata dates and +/// identifiers) so that the same source document always produces byte-identical snapshot output. +/// +/// +/// All edits are performed directly on the bytes and are length-preserving: only the mutable +/// characters inside each value are overwritten, so every cross-reference offset stays valid and +/// the file never has to be re-serialized. The scan is plaintext, so a value that has been +/// compressed away (inside an /ObjStm object stream or a flate-compressed XMP packet) no +/// longer appears literally and is therefore left as-is. +/// +/// Targets unencrypted documents. Encrypted PDFs seed their encryption key from the trailer +/// /ID; zeroing it would leave the document undecryptable, so encrypted input should not be +/// passed here. +/// +/// +static class PdfNormalizer +{ + enum Fill + { + // Zero the ASCII digits only, keeping separators (leaves a readable date). + Digits, + + // Zero the hexadecimal digits (for hex string <...> values). + Hex, + + // Zero every non-whitespace byte (for opaque identifiers). + All + } + + public static void Normalize(byte[] data) + { + // Document information dictionary dates. + ZeroPdfString(data, "/CreationDate"u8, Fill.Digits); + ZeroPdfString(data, "/ModDate"u8, Fill.Digits); + + // Trailer / cross-reference-stream file identifier: /ID [<...> <...>]. + ZeroFileId(data); + + // XMP metadata dates (uncompressed metadata streams only). + ZeroXmpElement(data, "". + static void ZeroPdfString(byte[] data, ReadOnlySpan key, Fill fill) + { + var pos = 0; + while (true) + { + var hit = data.AsSpan(pos).IndexOf(key); + if (hit < 0) + { + return; + } + + var i = pos + hit + key.Length; + pos = i; + + i = SkipWhitespace(data, i); + if (i >= data.Length) + { + return; + } + + if (data[i] == (byte) '(') + { + var start = i + 1; + var end = FindLiteralEnd(data, start); + Overwrite(data, start, end, fill); + pos = end; + } + else if (data[i] == (byte) '<' && (i + 1 >= data.Length || data[i + 1] != (byte) '<')) + { + var start = i + 1; + var end = FindByte(data, start, (byte) '>'); + Overwrite(data, start, end, Fill.Hex); + pos = end; + } + } + } + + // Finds "/ID" followed by an array and zeroes each string element. Anything not shaped like the + // identifier array (for example the "/IDTree" name-tree key) is skipped. + static void ZeroFileId(byte[] data) + { + var key = "/ID"u8; + var pos = 0; + while (true) + { + var hit = data.AsSpan(pos).IndexOf(key); + if (hit < 0) + { + return; + } + + var i = pos + hit + key.Length; + pos = i; + + i = SkipWhitespace(data, i); + if (i >= data.Length || data[i] != (byte) '[') + { + continue; + } + + i++; + while (i < data.Length && data[i] != (byte) ']') + { + if (data[i] == (byte) '<') + { + var start = i + 1; + i = FindByte(data, start, (byte) '>'); + Overwrite(data, start, i, Fill.Hex); + i++; + } + else if (data[i] == (byte) '(') + { + var start = i + 1; + i = FindLiteralEnd(data, start); + Overwrite(data, start, i, Fill.All); + i++; + } + else + { + i++; + } + } + + pos = i; + } + } + + // Finds an XMP element by its opening tag and zeroes the text content up to the next '<'. + static void ZeroXmpElement(byte[] data, ReadOnlySpan openTag, Fill fill) + { + var pos = 0; + while (true) + { + var hit = data.AsSpan(pos).IndexOf(openTag); + if (hit < 0) + { + return; + } + + var i = pos + hit + openTag.Length; + pos = i; + + // Reject a longer element name that merely shares this prefix. + if (i < data.Length && data[i] != (byte) '>' && data[i] != (byte) '/' && !IsWhitespace(data[i])) + { + continue; + } + + // Skip the remainder of the opening tag, remembering the last significant byte so a + // self-closing "" can be detected. + var lastSignificant = (byte) 0; + while (i < data.Length && data[i] != (byte) '>') + { + if (!IsWhitespace(data[i])) + { + lastSignificant = data[i]; + } + + i++; + } + + if (i >= data.Length) + { + return; + } + + i++; + if (lastSignificant == (byte) '/') + { + continue; + } + + var start = i; + var end = FindByte(data, start, (byte) '<'); + Overwrite(data, start, end, fill); + pos = end; + } + } + + static void Overwrite(byte[] data, int start, int end, Fill fill) + { + for (var i = start; i < end; i++) + { + var c = data[i]; + var replace = fill switch + { + Fill.Digits => IsDigit(c), + Fill.Hex => IsHexDigit(c), + _ => !IsWhitespace(c) + }; + if (replace) + { + data[i] = (byte) '0'; + } + } + } + + // Returns the index of the ')' that closes the literal string starting at 'start', honoring + // backslash escapes and balanced parentheses, or the end of the buffer if unterminated. + static int FindLiteralEnd(byte[] data, int start) + { + var depth = 1; + var i = start; + while (i < data.Length) + { + var c = data[i]; + if (c == (byte) '\\') + { + i += 2; + continue; + } + + if (c == (byte) '(') + { + depth++; + } + else if (c == (byte) ')') + { + depth--; + if (depth == 0) + { + return i; + } + } + + i++; + } + + return data.Length; + } + + static int FindByte(byte[] data, int start, byte target) + { + var i = start; + while (i < data.Length && data[i] != target) + { + i++; + } + + return i; + } + + static int SkipWhitespace(byte[] data, int i) + { + while (i < data.Length && IsWhitespace(data[i])) + { + i++; + } + + return i; + } + + static bool IsDigit(byte b) => + b is >= (byte) '0' and <= (byte) '9'; + + static bool IsHexDigit(byte b) => + b is >= (byte) '0' and <= (byte) '9' or >= (byte) 'a' and <= (byte) 'f' or >= (byte) 'A' and <= (byte) 'F'; + + static bool IsWhitespace(byte b) => + b is (byte) ' ' or (byte) '\t' or (byte) '\r' or (byte) '\n' or (byte) '\f' or 0; +} diff --git a/src/Verify.DocNet/VerifyDocNet_Pdf.cs b/src/Verify.DocNet/VerifyDocNet_Pdf.cs index cf19256..2c376bf 100644 --- a/src/Verify.DocNet/VerifyDocNet_Pdf.cs +++ b/src/Verify.DocNet/VerifyDocNet_Pdf.cs @@ -4,21 +4,50 @@ public static partial class VerifyDocNet { static ConversionResult Convert(string? name, Stream stream, IReadOnlyDictionary settings) { + var bytes = stream.ToBytes(); var dimensions = settings.GetPageDimensions(new(scalingFactor: 2)); - using var reader = DocLib.Instance.GetDocReader(stream.ToBytes(), dimensions); - return Convert(name, reader, settings); + List targets; + PdfInfo info; + int start; + int endExclusive; + using (var reader = DocLib.Instance.GetDocReader(bytes, dimensions)) + { + (targets, info, start, endExclusive) = Render(name, reader, settings); + } + + // Subset the source to only the rendered pages so the pdf snapshot matches the png/info + // pages. A full-document render reuses the original buffer; a filtered render is re-split + // via pdfium into a fresh buffer. + var coversAllPages = start == 0 && endExclusive == info.PageCount; + var pdfBytes = coversAllPages ? + bytes : + DocLib.Instance.Split(bytes, start, endExclusive - 1); + + // Neutralize the volatile fields for the pdf snapshot. When the whole document is reused + // this must happen only after the reader, which reads lazily from the same buffer, has been + // released. + PdfNormalizer.Normalize(pdfBytes); + targets.Add( + new("pdf", new MemoryStream(pdfBytes), "pdf") + { + BypassComparersForSubsequentOnDifference = true + }); + + return new(info, targets); } + // Registered for callers that supply an IDocReader directly. No pdf snapshot is produced here + // since the original bytes are not available to subset and normalize. static ConversionResult Convert(string? name, IDocReader document, IReadOnlyDictionary settings) { - var targets = GetStreams(name, document, settings).ToList(); - return new(null, targets); + var (targets, info, _, _) = Render(name, document, settings); + return new(info, targets); } static NaiveTransparencyRemover transparencyRemover = new(); - static IEnumerable GetStreams(string? name, IDocReader document, IReadOnlyDictionary settings) + static (List targets, PdfInfo info, int start, int endExclusive) Render(string? name, IDocReader document, IReadOnlyDictionary settings) { var numberOfPages = document.GetPageCount(); var pagesToInclude = settings.GetPagesToInclude(numberOfPages); @@ -36,6 +65,8 @@ static IEnumerable GetStreams(string? name, IDocReader document, IReadOn } var preserveTransparency = settings.GetPreserveTransparency(); + var targets = new List(); + var pages = new List(); for (var index = start; index < pagesToInclude; index++) { using var reader = document.GetPageReader(index); @@ -49,7 +80,17 @@ static IEnumerable GetStreams(string? name, IDocReader document, IReadOn var stream = new MemoryStream(); PngEncoder.WriteBgraAsPng(rawBytes, width, height, stream); - yield return new("png", stream, name); + targets.Add(new("png", stream, name)); + + pages.Add(new() { Index = index, Text = reader.GetText() }); } + + var info = new PdfInfo + { + Version = document.GetPdfVersion().ToString(), + PageCount = numberOfPages, + Pages = pages + }; + return (targets, info, start, pagesToInclude); } -} \ No newline at end of file +}