Deterministic pdf

readme.md

@@ -77,7 +77,7 @@ public Task VerifyPdfStream()
     return Verify(stream, "pdf");
 }
 ```
-<sup><a href='/src/Tests/Samples.cs#L29-L38' title='Snippet source file'>snippet source</a> | <a href='#snippet-VerifyPdfStream' title='Start of snippet'>anchor</a></sup>
+<sup><a href='/src/Tests/Samples.cs#L38-L47' title='Snippet source file'>snippet source</a> | <a href='#snippet-VerifyPdfStream' title='Start of snippet'>anchor</a></sup>
 <!-- endSnippet -->
 
 
@@ -98,7 +98,7 @@ public Task VerifyPreserveTransparency() =>
     VerifyFile("sample.pdf")
         .PreserveTransparency();
 ```
-<sup><a href='/src/Tests/Samples.cs#L11-L18' title='Snippet source file'>snippet source</a> | <a href='#snippet-PreserveTransparency' title='Start of snippet'>anchor</a></sup>
+<sup><a href='/src/Tests/Samples.cs#L20-L27' title='Snippet source file'>snippet source</a> | <a href='#snippet-PreserveTransparency' title='Start of snippet'>anchor</a></sup>
 <!-- endSnippet -->
 
 
@@ -112,7 +112,7 @@ public Task VerifyPageDimensions() =>
     VerifyFile("sample.pdf")
         .PageDimensions(new(1080, 1920));
 ```
-<sup><a href='/src/Tests/Samples.cs#L20-L27' title='Snippet source file'>snippet source</a> | <a href='#snippet-PageDimensions' title='Start of snippet'>anchor</a></sup>
+<sup><a href='/src/Tests/Samples.cs#L29-L36' title='Snippet source file'>snippet source</a> | <a href='#snippet-PageDimensions' title='Start of snippet'>anchor</a></sup>
 <!-- endSnippet -->
 
 ## VerifySinglePage
@@ -134,7 +134,7 @@ public Task VerifySecondPage()
     return Verify(stream, "pdf").SinglePage(1);
 }
 ```
-<sup><a href='/src/Tests/Samples.cs#L40-L56' title='Snippet source file'>snippet source</a> | <a href='#snippet-VerifySinglePage' title='Start of snippet'>anchor</a></sup>
+<sup><a href='/src/Tests/Samples.cs#L49-L65' title='Snippet source file'>snippet source</a> | <a href='#snippet-VerifySinglePage' title='Start of snippet'>anchor</a></sup>
 <!-- endSnippet -->
 
 ## File Samples

src/Directory.Build.props

@@ -2,7 +2,7 @@
 <Project>
   <PropertyGroup>
     <NoWarn>CS1591;CS0649;NU1608;NU1109</NoWarn>
-    <Version>3.3.2</Version>
+    <Version>3.5.0</Version>
     <AssemblyVersion>1.0.0</AssemblyVersion>
     <LangVersion>preview</LangVersion>
     <SignAssembly>false</SignAssembly>

src/Directory.Packages.props

@@ -4,6 +4,7 @@
     <CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
   </PropertyGroup>
   <ItemGroup>
+    <PackageVersion Include="DeterministicPdf" Version="1.0.2" />
     <PackageVersion Include="Docnet.Core" Version="2.6.0" />
     <PackageVersion Include="System.IO.Hashing" Version="10.0.10" />
     <PackageVersion Include="Magick.NET-Q16-AnyCPU" Version="14.15.0" />

src/Tests/FopTests.VerifyFopRenderedPdf.verified.txt

@@ -0,0 +1,11 @@
+{
+  Version: 1.4,
+  PageCount: 1,
+  Pages: [
+    {
+      Text:
+Verify.DocNet
+A deterministic PDF produced by Apache FOP for snapshot testing.
+    }
+  ]
+}

src/Tests/GlobalUsings.cs

@@ -1,4 +1,5 @@
-global using System.Buffers.Binary;
+global using DeterministicPdf;
+global using System.Buffers.Binary;
 global using System.Diagnostics;
 global using System.IO.Compression;
 global using Docnet.Core;

src/Tests/PdfNormalizerTests.cs

@@ -1,125 +1,12 @@
+// The neutralizing algorithm itself is owned and tested by the DeterministicPdf package. What is
+// worth asserting here is the wiring: that this package applies it, and that a normalized document
+// is still loadable by pdfium.
 public class PdfNormalizerTests
 {
-    [Test]
-    public async Task NeutralizesVolatileValues()
-    {
-        var input =
-            "/ID [<A1B2C3D4E5F60718> <1122334455667788>] " +
-            "/CreationDate(D:20240115093000+05'30') " +
-            "/ModDate(D:20240115093000Z) " +
-            "<xmp:CreateDate>2024-01-15T09:30:00+05:30</xmp:CreateDate>" +
-            "<xmp:ModifyDate>2024-01-15T09:30:00Z</xmp:ModifyDate>" +
-            "<xmp:MetadataDate>2024-01-15T09:30:00Z</xmp:MetadataDate>" +
-            "<xmpMM:DocumentID>uuid:0f7b2c9a-1234-5678-9abc-def012345678</xmpMM:DocumentID>" +
-            "<xmpMM:InstanceID>xmp.iid:1a2b3c4d</xmpMM:InstanceID>";
-        var expected =
-            "/ID [<0000000000000000> <0000000000000000>] " +
-            "/CreationDate(D:00000000000000+00'00') " +
-            "/ModDate(D:00000000000000Z) " +
-            "<xmp:CreateDate>0000-00-00T00:00:00+00:00</xmp:CreateDate>" +
-            "<xmp:ModifyDate>0000-00-00T00:00:00Z</xmp:ModifyDate>" +
-            "<xmp:MetadataDate>0000-00-00T00:00:00Z</xmp:MetadataDate>" +
-            $"<xmpMM:DocumentID>{new string('0', 41)}</xmpMM:DocumentID>" +
-            $"<xmpMM:InstanceID>{new string('0', 16)}</xmpMM:InstanceID>";
-        await Assert.That(Normalize(input)).IsEqualTo(expected);
-    }
-
-    [Test]
-    public async Task NeutralizesDublinCoreDate()
-    {
-        // Some producers (for example older Apache FOP) write the render time straight into the
-        // Dublin Core <dc:date> element as simple text content.
-        var input = "<dc:date>2024-01-15T09:30:00+05:30</dc:date>";
-        var expected = "<dc:date>0000-00-00T00:00:00+00:00</dc:date>";
-        await Assert.That(Normalize(input)).IsEqualTo(expected);
-    }
-
-    [Test]
-    public async Task NeutralizesDublinCoreDateSeq()
-    {
-        // Per the XMP spec dc:date is an ordered array (seq Date), so a spec-compliant producer
-        // (current Apache FOP) nests the render time in rdf:Seq/rdf:li rather than as direct text.
-        var input = "<dc:date><rdf:Seq><rdf:li>2024-01-15T09:30:00+05:30</rdf:li></rdf:Seq></dc:date>";
-        var expected = "<dc:date><rdf:Seq><rdf:li>0000-00-00T00:00:00+00:00</rdf:li></rdf:Seq></dc:date>";
-        await Assert.That(Normalize(input)).IsEqualTo(expected);
-    }
-
-    [Test]
-    public async Task NeutralizesDublinCoreDateSeqWithWhitespaceAndMultipleEntries()
-    {
-        // Pretty-printed with indentation and more than one date in the sequence: markup and
-        // whitespace are preserved while every date value is zeroed.
-        var input =
-            """
-            <dc:date>
-              <rdf:Seq>
-                <rdf:li>2024-01-15T09:30:00+05:30</rdf:li>
-                <rdf:li>2019-12-31T23:59:59Z</rdf:li>
-              </rdf:Seq>
-            </dc:date>
-            """;
-        var expected =
-            """
-            <dc:date>
-              <rdf:Seq>
-                <rdf:li>0000-00-00T00:00:00+00:00</rdf:li>
-                <rdf:li>0000-00-00T00:00:00Z</rdf:li>
-              </rdf:Seq>
-            </dc:date>
-            """;
-        await Assert.That(Normalize(input)).IsEqualTo(expected);
-    }
-
-    [Test]
-    public async Task LeavesNonDateRdfArraysUntouched()
-    {
-        // The rdf:li descent is scoped to dc:date, so digits in a sibling array (here dc:subject)
-        // must survive.
-        var input =
-            "<dc:subject><rdf:Bag><rdf:li>topic 2024</rdf:li></rdf:Bag></dc:subject>" +
-            "<dc:date><rdf:Seq><rdf:li>2024-01-15T09:30:00Z</rdf:li></rdf:Seq></dc:date>";
-        var expected =
-            "<dc:subject><rdf:Bag><rdf:li>topic 2024</rdf:li></rdf:Bag></dc:subject>" +
-            "<dc:date><rdf:Seq><rdf:li>0000-00-00T00:00:00Z</rdf:li></rdf:Seq></dc:date>";
-        await Assert.That(Normalize(input)).IsEqualTo(expected);
-    }
-
-    [Test]
-    public async Task CollapsesDifferingValuesToTheSameOutput()
-    {
-        // The same producer emits a stable structure across runs, so two documents differing only
-        // in the volatile digits/hex normalize to identical bytes.
-        var a = "/ID [<A1B2C3D4>] /CreationDate(D:20240115093000+05'30')";
-        var b = "/ID [<99887766>] /CreationDate(D:19991231235959+11'45')";
-        await Assert.That(a).IsNotEqualTo(b);
-        await Assert.That(Normalize(a)).IsEqualTo(Normalize(b));
-    }
-
-    [Test]
-    public async Task LeavesLookalikeKeysUntouched()
-    {
-        // /IDTree is a name-tree key (not the file identifier), /ModDateStamp is a different name,
-        // and a self-closing date element has no content: none should be altered.
-        var input = "/IDTree [1 2] /ModDateStamp(20240101) <xmp:CreateDate/>2024";
-        await Assert.That(Normalize(input)).IsEqualTo(input);
-    }
-
-    [Test]
-    public async Task HandlesUnterminatedFileIdWithoutOverrunning()
-    {
-        // A truncated /ID whose string runs to the end of the buffer with no closing '>'/')' (and no
-        // closing ']') must not scan past the buffer: the value is zeroed as far as it exists and the
-        // scan stops cleanly rather than throwing. Hex string form (no closing '>'):
-        await Assert.That(Normalize("/ID [<ABCD")).IsEqualTo("/ID [<0000");
-        // Literal string form (no closing ')'):
-        await Assert.That(Normalize("/ID [(ABCD")).IsEqualTo("/ID [(0000");
-    }
-
     [Test]
     public async Task NormalizedDocumentStillLoads()
     {
-        var data = File.ReadAllBytes("sample.pdf");
-        PdfNormalizer.Normalize(data);
+        var data = PdfNormalizer.Normalize(await File.ReadAllBytesAsync("sample.pdf"));
 
         using var reader = DocLib.Instance.GetDocReader(data, new(scalingFactor: 2));
         await Assert.That(reader.GetPageCount()).IsEqualTo(2);
@@ -130,8 +17,7 @@ public async Task NeutralizesFopStyleXmp()
     {
         // sample-fop.pdf carries an uncompressed FOP-style XMP packet whose dc:date render time is
         // nested in rdf:Seq/rdf:li. It must be neutralized while the document still loads.
-        var data = File.ReadAllBytes("sample-fop.pdf");
-        PdfNormalizer.Normalize(data);
+        var data = PdfNormalizer.Normalize(await File.ReadAllBytesAsync("sample-fop.pdf"));
 
         var text = Encoding.Latin1.GetString(data);
         await Assert.That(text).Contains("<rdf:li>0000-00-00T00:00:00+00:00</rdf:li>");
@@ -142,33 +28,34 @@ public async Task NeutralizesFopStyleXmp()
     }
 
     [Test]
-    public async Task IsIdempotent()
+    public async Task CanonicalizesXmpAcrossJreSerializers()
     {
-        // A second pass has nothing left to change: normalizing already-normalized bytes is a no-op.
-        var once = File.ReadAllBytes("sample.pdf");
-        PdfNormalizer.Normalize(once);
-        var twice = (byte[])once.Clone();
-        PdfNormalizer.Normalize(twice);
-        await Assert.That(twice).IsEquivalentTo(once);
+        // The same FOP document rendered on two machines. Apache FOP serializes the XMP packet through
+        // the platform's XML writer, so the JDK decides the indentation: one build emits a compact
+        // packet, the other indents every element, and the raw bytes differ. Once normalized, the pdf
+        // snapshot must collapse to identical bytes on both, and still load.
+        var compactRaw = await File.ReadAllBytesAsync("sample-fop-compact.pdf");
+        var indentedRaw = await File.ReadAllBytesAsync("sample-fop-indented.pdf");
+        await Assert.That(compactRaw.SequenceEqual(indentedRaw)).IsFalse();
+
+        var compact = PdfNormalizer.Normalize(compactRaw);
+        var indented = PdfNormalizer.Normalize(indentedRaw);
+        await Assert.That(compact).IsEquivalentTo(indented);
+
+        using var reader = DocLib.Instance.GetDocReader(compact, new(scalingFactor: 2));
+        await Assert.That(reader.GetPageCount()).IsEqualTo(1);
     }
 
     [Test]
     public async Task NormalizedSinglePageSplitStillLoads()
     {
         // A page subset is re-serialized by pdfium (reintroducing volatile fields) then normalized;
         // it must remain a valid one-page document.
-        var data = File.ReadAllBytes("sample.pdf");
+        var data = await File.ReadAllBytesAsync("sample.pdf");
         var split = DocLib.Instance.Split(data, 1, 1);
-        PdfNormalizer.Normalize(split);
+        split = PdfNormalizer.Normalize(split);
 
         using var reader = DocLib.Instance.GetDocReader(split, new(scalingFactor: 2));
         await Assert.That(reader.GetPageCount()).IsEqualTo(1);
     }
-
-    static string Normalize(string value)
-    {
-        var bytes = Encoding.Latin1.GetBytes(value);
-        PdfNormalizer.Normalize(bytes);
-        return Encoding.Latin1.GetString(bytes);
-    }
 }

src/Tests/Samples.SkipPdfNormalization.verified.txt

@@ -0,0 +1,29 @@
+{
+  Version: 1.4,
+  PageCount: 2,
+  Pages: [
+    {
+      Text:
+ A Simple PDF File 
+ This is a small demonstration .pdf file - 
+ just for use in the Virtual Mechanics tutorials. More text. And more 
+ text. And more text. And more text. And more text. 
+ And more text. And more text. And more text. And more text. And more 
+ text. And more text. Boring, zzzzz. And more text. And more text. And 
+ more text. And more text. And more text. And more text. And more text. 
+ And more text. And more text. 
+ And more text. And more text. And more text. And more text. And more 
+ text. And more text. And more text. Even more. Continued on page 2 ...
+    },
+    {
+      Index: 1,
+      Text:
+ Simple PDF File 2 
+ ...continued from page 1. Yet more text. And more text. And more text. 
+ And more text. And more text. And more text. And more text. And more 
+ text. Oh, how boring typing this stuff. But not as boring as watching 
+ paint dry. And more text. And more text. And more text. And more text. 
+ Boring. More, a little more text. The end, and just as well. 
+    }
+  ]
+}

src/Tests/Samples.cs

@@ -8,6 +8,15 @@ public Task VerifyPdf() =>
 
     #endregion
 
+    #region SkipPdfNormalization
+
+    [Test]
+    public Task SkipPdfNormalization() =>
+        VerifyFile("sample.pdf")
+            .SkipPdfNormalization();
+
+    #endregion
+
     #region PreserveTransparency
 
     [Test]

src/Tests/SkipPdfNormalizationTests.NormalizedSnapshotHoldsTheNeutralizedBytes.verified.txt

@@ -0,0 +1,29 @@
+{
+  Version: 1.4,
+  PageCount: 2,
+  Pages: [
+    {
+      Text:
+ A Simple PDF File 
+ This is a small demonstration .pdf file - 
+ just for use in the Virtual Mechanics tutorials. More text. And more 
+ text. And more text. And more text. And more text. 
+ And more text. And more text. And more text. And more text. And more 
+ text. And more text. Boring, zzzzz. And more text. And more text. And 
+ more text. And more text. And more text. And more text. And more text. 
+ And more text. And more text. 
+ And more text. And more text. And more text. And more text. And more 
+ text. And more text. And more text. Even more. Continued on page 2 ...
+    },
+    {
+      Index: 1,
+      Text:
+ Simple PDF File 2 
+ ...continued from page 1. Yet more text. And more text. And more text. 
+ And more text. And more text. And more text. And more text. And more 
+ text. Oh, how boring typing this stuff. But not as boring as watching 
+ paint dry. And more text. And more text. And more text. And more text. 
+ Boring. More, a little more text. The end, and just as well. 
+    }
+  ]
+}

src/Tests/SkipPdfNormalizationTests.SkippedSnapshotHoldsTheProducerBytes.verified.txt

@@ -0,0 +1,29 @@
+{
+  Version: 1.4,
+  PageCount: 2,
+  Pages: [
+    {
+      Text:
+ A Simple PDF File 
+ This is a small demonstration .pdf file - 
+ just for use in the Virtual Mechanics tutorials. More text. And more 
+ text. And more text. And more text. And more text. 
+ And more text. And more text. And more text. And more text. And more 
+ text. And more text. Boring, zzzzz. And more text. And more text. And 
+ more text. And more text. And more text. And more text. And more text. 
+ And more text. And more text. 
+ And more text. And more text. And more text. And more text. And more 
+ text. And more text. And more text. Even more. Continued on page 2 ...
+    },
+    {
+      Index: 1,
+      Text:
+ Simple PDF File 2 
+ ...continued from page 1. Yet more text. And more text. And more text. 
+ And more text. And more text. And more text. And more text. And more 
+ text. Oh, how boring typing this stuff. But not as boring as watching 
+ paint dry. And more text. And more text. And more text. And more text. 
+ Boring. More, a little more text. The end, and just as well. 
+    }
+  ]
+}

src/Tests/SkipPdfNormalizationTests.cs

@@ -0,0 +1,24 @@
+// The Samples entry for SkipPdfNormalization only proves the setting round-trips through a
+// verification. These assert the wiring it is actually there for: with the setting the snapshotted
+// bytes are the producer's own, and without it they are the normalized ones.
+public class SkipPdfNormalizationTests
+{
+    [Test]
+    public async Task SampleIsNotAlreadyNormalized()
+    {
+        // The premise the two tests below rest on. If sample.pdf were already byte-identical to its
+        // normalized form they would both pass while asserting nothing.
+        var raw = await File.ReadAllBytesAsync("sample.pdf");
+
+        await Assert.That(PdfNormalizer.Normalize(raw)).IsNotEquivalentTo(raw);
+    }
+
+    [Test]
+    public async Task SkippedSnapshotHoldsTheProducerBytes() =>
+        await Verify(new MemoryStream(await File.ReadAllBytesAsync("sample.pdf")), "pdf")
+            .SkipPdfNormalization();
+
+    [Test]
+    public async Task NormalizedSnapshotHoldsTheNeutralizedBytes() =>
+        await Verify(new MemoryStream(await File.ReadAllBytesAsync("sample.pdf")), "pdf");
+}