diff --git a/CHANGELOG.md b/CHANGELOG.md
index 99b7fa4..4d058c2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -38,6 +38,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
of the document, and fails instead with `PclReadOptions.Strict`.
- The point count of a PinPoint location of a manifest with a data object list is taken from the data file the object
refers to, as for manifests with a scan position list. Locations that share one file each report its full count.
+- The Snappy decoder now keeps a window of up to 8 MiB per stream (the declared uncompressed length, capped), instead of
+ a fixed 1 MiB, so streams with copies further back decode within bounded memory.
## [0.1.0] - 2026-11-15
diff --git a/README.md b/README.md
index 0b95d8c..a887595 100644
--- a/README.md
+++ b/README.md
@@ -93,7 +93,7 @@ These readers drop data without an error or a warning, so a scan can yield fewer
A full read reports how many points it dropped in the warnings of the document, and fails instead with `PclReadOptions.Strict`. A
data file in neither layout, or in the sectioned layout with an element type or header value the reader does not support, cannot
be read: the project lists it with a warning and reading its points throws `NotSupportedException`. A compressed stream whose
- copies reach back more than 1 MiB is not decoded either: reading its points throws `NotSupportedException`, without an earlier warning.
+ copies reach back more than 8 MiB is not decoded: reading its points throws `NotSupportedException`, without an earlier warning.
- Leica: padding entries and points that coincide with a block's origin are dropped, a compressed stream above its size limit is
skipped, and a block that is not recognised ends the scan. The reader counts what it dropped or skipped and reports it in the
warnings of the document, so that an incomplete result is visible. With `PclReadOptions.Strict` the read fails instead.
@@ -118,7 +118,7 @@ outside the file is fetched.
| ReCap scan | 500,000 nodes in the node table, which must fit in the file together with the node payloads. |
| Leica | 10,000 container entries; per point block at most 64 compressed streams, 10,000,000 compressed bytes per stream, 16 MiB per decompressed stream (checked before allocation) and 128 MiB decompressed in all; over all blocks of one read 1 GiB plus 64 times the file size decompressed. |
| FARO | Parameters 1 MiB; scan grid of at most 2^27 cells (134,217,728); a distance stream too small for the declared grid is refused; each compressed data stream is held in memory (see above) and is inflated in chunks, and must not inflate beyond its grid size plus 64 KiB. Compound file: 65,536 directory entries, depth 16. |
-| PinPoint | Manifest 64 MiB and the default nesting depth of `System.Text.Json` (64); 100,000 scans. The parsed manifest needs memory of about 15 times its size, so a manifest at the limit needs about 1 GB. Sectioned data file: at most 16 sections, a property table of at most 1 MiB nested at most 8 levels, copies in a compressed stream at most 1 MiB back; the sections must be contiguous and fill the file, and the `Points`, `RGB8` and `Intensities8` streams must decode to exactly 12, 3 and 1 bytes per stored point. |
+| PinPoint | Manifest 64 MiB and the default nesting depth of `System.Text.Json` (64); 100,000 scans. The parsed manifest needs memory of about 15 times its size, so a manifest at the limit needs about 1 GB. Sectioned data file: at most 16 sections, a property table of at most 1 MiB nested at most 8 levels, copies in a compressed stream at most 8 MiB back with a decoder window of at most 8 MiB per stream; the sections must be contiguous and fill the file, and the `Points`, `RGB8`, `Intensities8`, `Normals` and `FilterBits` streams must decode to exactly 12, 3, 1, 12 and 1 bytes per stored point. |
| EdgeWise | 1,048,576 characters per database line; 1,000,000 nodes. Patterns that parse a line have a time limit. |
## Exporters
diff --git a/src/PclIO/Internal/SnappyReader.cs b/src/PclIO/Internal/SnappyReader.cs
index d79f047..e96bae8 100644
--- a/src/PclIO/Internal/SnappyReader.cs
+++ b/src/PclIO/Internal/SnappyReader.cs
@@ -4,21 +4,23 @@ namespace PclIO.Internal;
///
/// Decodes a raw Snappy stream incrementally: a varint holding the uncompressed length, followed by literal and copy
-/// elements. Only the last decoded bytes are kept, so memory use does not depend on the size of
-/// the data. The reader owns source and disposes it.
+/// elements. Only the last decoded bytes up to the window are kept: the window is the declared uncompressed length
+/// rounded up to a power of two, but never more than , so memory use stays bounded.
+/// A stream whose copies reach back further than the window cannot be decoded and is reported as unsupported instead
+/// of being decoded wrongly. The reader owns source and disposes it.
///
internal sealed class SnappyReader : Stream
{
- // Copies may reach back at most this far. Encoders that split their input into 64 KiB fragments never exceed it;
- // a stream that does is reported as unsupported instead of being decoded wrongly.
- private const int WindowSize = 1 << 20;
- private const int WindowMask = WindowSize - 1;
+ // The window never grows beyond this, however long the stream declares to be.
+ private const int MaxWindowBytes = 8 << 20;
private const int InputSize = 64 * 1024;
private const int MaxPreambleBytes = 5;
private readonly Stream _source;
private readonly byte[] _input = ArrayPool.Shared.Rent(InputSize);
- private readonly byte[] _window = ArrayPool.Shared.Rent(WindowSize);
+ private readonly byte[] _window;
+ private readonly int _windowSize;
+ private readonly int _windowMask;
private long _sourceLeft;
private int _inputPos, _inputEnd;
private long _produced;
@@ -35,14 +37,35 @@ internal sealed class SnappyReader : Stream
public SnappyReader(Stream source, long compressedLength)
{
_source = source;
+
+ // The preamble is read before anything is rented, so its failure only needs the source disposed.
+ Span head = stackalloc byte[MaxPreambleBytes];
+ int want = (int)Math.Min(MaxPreambleBytes, compressedLength);
+ int got;
+ int used;
+ long length;
try
{
- Span head = stackalloc byte[MaxPreambleBytes];
- int want = (int)Math.Min(MaxPreambleBytes, compressedLength);
- int got = source.ReadAtLeast(head[..want], want, throwOnEndOfStream: false);
- int used = ParsePreamble(head[..got], out long length);
- UncompressedLength = length;
+ got = source.ReadAtLeast(head[..want], want, throwOnEndOfStream: false);
+ used = ParsePreamble(head[..got], out length);
+ }
+ catch
+ {
+ source.Dispose();
+ throw;
+ }
+
+ UncompressedLength = length;
+ // The window holds every byte the stream can copy from, unless the stream is longer than the cap.
+ long windowBytes = 1;
+ while (windowBytes < length && windowBytes < MaxWindowBytes) windowBytes <<= 1;
+ _window = ArrayPool.Shared.Rent((int)windowBytes);
+ _windowSize = (int)windowBytes;
+ _windowMask = (int)windowBytes - 1;
+
+ try
+ {
// Bytes read past the preamble are the start of the first element.
head[used..got].CopyTo(_input);
_inputEnd = got - used;
@@ -104,8 +127,8 @@ public override int Read(Span buffer)
for (int i = 0; i < n; i++)
{
// A copy may overlap its own output, so it proceeds byte by byte.
- byte value = _window[(int)((_produced - _copyOffset) & WindowMask)];
- _window[(int)(_produced & WindowMask)] = value;
+ byte value = _window[(int)((_produced - _copyOffset) & _windowMask)];
+ _window[(int)(_produced & _windowMask)] = value;
buffer[written + i] = value;
_produced++;
}
@@ -175,7 +198,8 @@ private bool NextElement()
}
if (offset == 0 || offset > _produced) throw FormatError.Corrupted("a copy reaches before the start of the data");
- if (offset > WindowSize) throw FormatError.Unsupported($"a compressed stream copies from more than {WindowSize} bytes back");
+ if (offset > _windowSize) throw FormatError.Unsupported(
+ $"a compressed stream copies from more than {_windowSize} bytes back; the decoder keeps at most {MaxWindowBytes} bytes");
if (copyLength > remaining) throw FormatError.Corrupted("a copy runs past the declared length");
_copyLeft = copyLength;
_copyOffset = (int)offset;
@@ -184,8 +208,8 @@ private bool NextElement()
private void StoreInWindow(ReadOnlySpan data)
{
- int position = (int)(_produced & WindowMask);
- int first = Math.Min(data.Length, WindowSize - position);
+ int position = (int)(_produced & _windowMask);
+ int first = Math.Min(data.Length, _windowSize - position);
data[..first].CopyTo(_window.AsSpan(position));
data[first..].CopyTo(_window);
}
diff --git a/tests/PclIO.Tests/PinPointPackedTests.cs b/tests/PclIO.Tests/PinPointPackedTests.cs
index 4cd017d..e5a25b1 100644
--- a/tests/PclIO.Tests/PinPointPackedTests.cs
+++ b/tests/PclIO.Tests/PinPointPackedTests.cs
@@ -90,6 +90,27 @@ public void A_Normals_Channel_With_The_Wrong_Size_Is_Rejected()
Assert.Throws(() => ReadAll(doc));
}
+ [Fact]
+ public void A_Points_Channel_With_A_Copy_Further_Back_Than_One_Mebibyte_Is_Decoded()
+ {
+ // A copy that reaches 2 MiB back: beyond the old 1 MiB window, inside the new one.
+ const int points = 200_000;
+ var pointsStream = new SnappyBuilder().Literal(new byte[2_100_000]);
+ for (int done = 0; done < 300_000; done += 64) pointsStream.Copy(2_000_000, Math.Min(64, 300_000 - done), 4);
+ var bin = PinPointPackedFixtures.Assemble(
+ [
+ ("RowColIndex", PinPointPackedFixtures.Literals(new byte[points * 8])),
+ ("Points", pointsStream.Build()),
+ ("FilterBits", PinPointPackedFixtures.Literals(new byte[points])),
+ ], points);
+ using var dir = new TempDirectory();
+ using var doc = Open(dir, bin);
+
+ var scan = (PinPointScan)Assert.Single(doc.Scans);
+ Assert.Equal(points, scan.PointCount);
+ Assert.Equal(points, ReadAll(doc).Count);
+ }
+
[Fact]
public void Flagged_Points_Are_Returned_By_Default_And_Excluded_On_Request()
{
diff --git a/tests/PclIO.Tests/SnappyReaderTests.cs b/tests/PclIO.Tests/SnappyReaderTests.cs
index 318dc00..4912136 100644
--- a/tests/PclIO.Tests/SnappyReaderTests.cs
+++ b/tests/PclIO.Tests/SnappyReaderTests.cs
@@ -106,15 +106,28 @@ public void The_Result_Does_Not_Depend_On_How_Many_Bytes_Each_Read_Asks_For(int
}
[Fact]
- public void A_Copy_Can_Reach_Back_One_Mebibyte_Across_The_Window_Wrap()
+ public void A_Copy_Can_Reach_Back_Several_Mebibytes()
{
- byte[] data = Random((1 << 20) + 1000, 2);
+ byte[] data = Random((2 << 20) + 1000, 2);
var expected = new List(data);
var builder = new SnappyBuilder().Literal(data);
- builder.Copy(1 << 20, 64, 4);
- ExpandCopy(expected, 1 << 20, 64);
- builder.Copy(1 << 20, 64, 4);
- ExpandCopy(expected, 1 << 20, 64);
+ builder.Copy(2 << 20, 64, 4);
+ ExpandCopy(expected, 2 << 20, 64);
+ builder.Copy(2 << 20, 64, 4);
+ ExpandCopy(expected, 2 << 20, 64);
+
+ Assert.Equal(expected, DecodeAll(builder.Build()));
+ }
+
+ [Fact]
+ public void Copies_Stay_Correct_When_The_Window_Wraps()
+ {
+ // The stream declares more than the capped window, so the window wraps while an old copy is still in reach.
+ byte[] data = new byte[8 << 20];
+ var expected = new List(data);
+ var builder = new SnappyBuilder().Literal(data);
+ builder.Copy(8 << 20, 64, 4);
+ ExpandCopy(expected, 8 << 20, 64);
Assert.Equal(expected, DecodeAll(builder.Build()));
}
@@ -122,8 +135,8 @@ public void A_Copy_Can_Reach_Back_One_Mebibyte_Across_The_Window_Wrap()
[Fact]
public void A_Copy_Reaching_Further_Back_Than_The_Window_Is_Reported_As_Unsupported()
{
- byte[] data = Random((1 << 20) + 1000, 3);
- byte[] stream = new SnappyBuilder().Literal(data).Copy((1 << 20) + 1, 64, 4).Build();
+ byte[] data = Random((8 << 20) + 1000, 3);
+ byte[] stream = new SnappyBuilder().Literal(data).Copy((8 << 20) + 1, 64, 4).Build();
Assert.Throws(() => DecodeAll(stream));
}