Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
integer are decoded; other shapes stay unread.
- The E57 reader decodes plain bit-packing only and names any other codec of the scan in the rejection message. The point
prototype limit is the packet header bound of 65535 fields.
- The Zstandard decoder is faster on Huffman-heavy data: the four Huffman streams of a block now decode interleaved
instead of one after another, so independent bit readers overlap in the processor; the per-symbol mask is hoisted out
of the bit reader loop. No unsafe code, no weakened checks; the benchmark in `ZstdBenchmarkTests` shows the speed
next to the reference decoder (about 1.7 times slower on the benchmark payload, down from about 2).
- The EdgeWise cluster position scale is now `PclReadOptions.EdgeWiseCoordinateScale`, so the caller states it; the
default keeps the long-standing assumption (32768 units = 0.128 m).
- The LAS exporter writes 1.4 with point formats 6 to 8 (`PclLasOptions` selects version, format, scale and offset);
Expand Down
1 change: 1 addition & 0 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.12.0" />
<PackageVersion Include="xunit" Version="2.9.3" />
<PackageVersion Include="xunit.runner.visualstudio" Version="4.0.1" />
<PackageVersion Include="ZstdSharp.Port" Version="0.8.8" />
</ItemGroup>

</Project>
3 changes: 2 additions & 1 deletion THIRD-PARTY-NOTICES.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

PclIO has no third-party runtime components: the published packages depend on no other package, and no third-party source code or binaries are included. The Zstandard and Snappy decoders it needs are its own code, written from the public format specifications (RFC 8878 for Zstandard, including its XXH64 content checksum, and the Snappy format description).

Development and build-time packages (xunit, xunit.runner.visualstudio, Microsoft.NET.Test.Sdk, Microsoft.SourceLink.GitHub) are not part of the published packages.
Development and build-time packages (xunit, xunit.runner.visualstudio, Microsoft.NET.Test.Sdk, Microsoft.SourceLink.GitHub,
and ZstdSharp.Port, a test-only benchmark reference for the Zstandard decoder) are not part of the published packages.

ASTM E57, ASPRS LAS, Zstandard, Snappy and the compound file (OLE2) container are names of file formats and standards. They are used only to say which formats PclIO reads or writes.

Expand Down
13 changes: 13 additions & 0 deletions src/PclIO/Internal/ZstdBitReaders.cs
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,17 @@ public uint Peek(int bits)
return (uint)(value & ((1UL << bits) - 1));
}

/// <summary>Returns the next <paramref name="bits"/> bits (at most 32) without consuming them, masking with a
/// caller-provided mask of <c>(1 &lt;&lt; bits) - 1</c> so that hot loops do not recompute it per symbol.</summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public uint PeekMasked(int bits, ulong mask)
{
if (_count < bits) Refill();
int shift = _count - bits;
ulong value = shift >= 0 ? _container >> shift : _container << -shift;
return (uint)(value & mask);
}

/// <summary>Consumes bits that a preceding <see cref="Peek"/> returned.</summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Skip(int bits) => _count -= bits;
Expand All @@ -98,6 +109,8 @@ public uint Read(int bits)
return value;
}

// The refill path stays out of line so that the hot Peek and Read stay small enough to inline.
[MethodImpl(MethodImplOptions.NoInlining)]
private void Refill()
{
if (_bytePosition >= 8)
Expand Down
12 changes: 7 additions & 5 deletions src/PclIO/Internal/ZstdDecoder.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Buffers;
using System.Runtime.CompilerServices;

namespace PclIO.Internal;

Expand Down Expand Up @@ -416,10 +417,8 @@ private int ReadLiterals(ReadOnlySpan<byte> block, int maxBlock, out int section
if (size4 < 0 || regenerated - 3 * quarter < 0) throw FormatError.Corrupted("Zstandard Huffman stream sizes are inconsistent.");

ReadOnlySpan<byte> body = content[6..];
_huffman.DecodeStream(body[..size1], literals[..quarter]);
_huffman.DecodeStream(body.Slice(size1, size2), literals.Slice(quarter, quarter));
_huffman.DecodeStream(body.Slice(size1 + size2, size3), literals.Slice(2 * quarter, quarter));
_huffman.DecodeStream(body[(size1 + size2 + size3)..], literals[(3 * quarter)..]);
_huffman.DecodeInterleaved(body[..size1], body.Slice(size1, size2), body.Slice(size1 + size2, size3), body[(size1 + size2 + size3)..],
literals[..quarter], literals.Slice(quarter, quarter), literals.Slice(2 * quarter, quarter), literals[(3 * quarter)..]);
return regenerated;
}

Expand All @@ -443,6 +442,8 @@ private void ExecuteSequences(ReadOnlySpan<byte> stream, int sequenceCount, int
int position = _length;
int literalPosition = 0;
uint repeat1 = _repeat1, repeat2 = _repeat2, repeat3 = _repeat3;
int frameStart = _frameStart;
long windowSize = _windowSize;

for (int i = 0; i < sequenceCount; i++)
{
Expand Down Expand Up @@ -505,7 +506,7 @@ private void ExecuteSequences(ReadOnlySpan<byte> stream, int sequenceCount, int
}
}

if (offset == 0 || offset > position - _frameStart || offset > _windowSize)
if (offset == 0 || offset > position - frameStart || offset > windowSize)
throw FormatError.Corrupted("Zstandard match offset reaches outside the window.");

CopyMatch(output, position, (int)offset, matchLength);
Expand All @@ -524,6 +525,7 @@ private void ExecuteSequences(ReadOnlySpan<byte> stream, int sequenceCount, int
}

/// <summary>Copies a match that may overlap its own output: a distance shorter than the length repeats the pattern.</summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void CopyMatch(Span<byte> output, int position, int offset, int length)
{
int source = position - offset;
Expand Down
67 changes: 66 additions & 1 deletion src/PclIO/Internal/ZstdHuffman.cs
Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,81 @@ public void DecodeStream(ReadOnlySpan<byte> stream, Span<byte> destination)
var bits = new ZstdBackwardBits(stream);
ushort[] table = _table;
int tableBits = _tableBits;
ulong mask = (1UL << tableBits) - 1;
for (int i = 0; i < destination.Length; i++)
{
ushort entry = table[bits.Peek(tableBits)];
ushort entry = table[bits.PeekMasked(tableBits, mask)];
destination[i] = (byte)(entry >> 4);
bits.Skip(entry & 0xF);
}

if (!bits.IsExhausted) throw FormatError.Corrupted("Zstandard Huffman stream was not consumed exactly.");
}

/// <summary>
/// Decodes four interleaved Huffman streams, round-robin, so that independent bit readers overlap in the
/// processor: where the single-stream loop waits on each bit read, this one decodes four symbols per pass.
/// The first three destinations hold a quarter of the symbols each, the fourth the rest.
/// </summary>
public void DecodeInterleaved(ReadOnlySpan<byte> first, ReadOnlySpan<byte> second, ReadOnlySpan<byte> third, ReadOnlySpan<byte> fourth,
Span<byte> firstOut, Span<byte> secondOut, Span<byte> thirdOut, Span<byte> fourthOut)
{
var b0 = new ZstdBackwardBits(first);
var b1 = new ZstdBackwardBits(second);
var b2 = new ZstdBackwardBits(third);
var b3 = new ZstdBackwardBits(fourth);
ushort[] table = _table;
int tableBits = _tableBits;
ulong mask = (1UL << tableBits) - 1;

int i = 0;
int rest = fourthOut.Length;
while (i < rest)
{
ushort e0 = table[b0.PeekMasked(tableBits, mask)];
firstOut[i] = (byte)(e0 >> 4);
b0.Skip(e0 & 0xF);

ushort e1 = table[b1.PeekMasked(tableBits, mask)];
secondOut[i] = (byte)(e1 >> 4);
b1.Skip(e1 & 0xF);

ushort e2 = table[b2.PeekMasked(tableBits, mask)];
thirdOut[i] = (byte)(e2 >> 4);
b2.Skip(e2 & 0xF);

ushort e3 = table[b3.PeekMasked(tableBits, mask)];
fourthOut[i] = (byte)(e3 >> 4);
b3.Skip(e3 & 0xF);

i++;
}

for (int j = rest; j < firstOut.Length; j++)
{
ushort entry = table[b0.PeekMasked(tableBits, mask)];
firstOut[j] = (byte)(entry >> 4);
b0.Skip(entry & 0xF);
}

for (int j = rest; j < secondOut.Length; j++)
{
ushort entry = table[b1.PeekMasked(tableBits, mask)];
secondOut[j] = (byte)(entry >> 4);
b1.Skip(entry & 0xF);
}

for (int j = rest; j < thirdOut.Length; j++)
{
ushort entry = table[b2.PeekMasked(tableBits, mask)];
thirdOut[j] = (byte)(entry >> 4);
b2.Skip(entry & 0xF);
}

if (!b0.IsExhausted || !b1.IsExhausted || !b2.IsExhausted || !b3.IsExhausted)
throw FormatError.Corrupted("Zstandard Huffman stream was not consumed exactly.");
}

/// <summary>Decodes the weights with two interleaved FSE states that share one table, until the stream runs dry.</summary>
private int DecodeWeights(ReadOnlySpan<byte> stream)
{
Expand Down
1 change: 1 addition & 0 deletions tests/PclIO.Tests/PclIO.Tests.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" />
<PackageReference Include="xunit" />
<PackageReference Include="ZstdSharp.Port" />
<PackageReference Include="xunit.runner.visualstudio">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
Expand Down
104 changes: 104 additions & 0 deletions tests/PclIO.Tests/ZstdBenchmarkTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
using System.Diagnostics;
using PclIO.Internal;
using Xunit;
using Xunit.Abstractions;

namespace PclIO.Tests;

/// <summary>Benchmark for the managed Zstandard decoder: decodes a large concatenated payload made of reference
/// compressor frames, asserts the output and reports the throughput. The output (not the time) is asserted, so the
/// test stays green on any machine; run it in Release and read the MB/s from the test output.</summary>
public sealed class ZstdBenchmarkTests
{
private readonly ITestOutputHelper _output;

public ZstdBenchmarkTests(ITestOutputHelper output) => _output = output;

// Large frames that together exercise raw blocks, RLE blocks, Huffman literals (single and four streams),
// treeless blocks, repeat offsets and far matches.
private static readonly string[] PayloadFrames =
[
"MultiBlock", "MultiBlockLevel19", "TreelessLiterals", "HuffmanLargeLiterals",
"RepeatOffsets", "FarMatch", "TextLevel19", "MixedContent",
];

private static byte[] FrameOf(string name) =>
Convert.FromBase64String(ZstdVectors.All.Single(v => v.Name == name).Base64);

[Fact]
public void Decode_Throughput_Of_A_Large_Concatenated_Payload()
{
#if DEBUG
const int repeats = 1;
const int timedRuns = 1;
#else
const int repeats = 144;
const int timedRuns = 5;
#endif
byte[][] frames = [.. PayloadFrames.Select(FrameOf)];
byte[] input = [.. frames.SelectMany(f => Enumerable.Repeat(f, repeats)).SelectMany(f => f)];

// The payload concatenates whole frames, so the content is the frames' contents one after another.
var contents = new List<byte>();
foreach (string name in PayloadFrames)
for (int i = 0; i < repeats; i++)
contents.AddRange(ZstdDecoderTests.ContentOf(name));
byte[] expected = [.. contents];

// Correctness first: the benchmark is only meaningful when the output is right.
Assert.Equal(expected, ZstdDecoder.Decompress(input, expected.Length));

// The reference decoder (a C# port of zstd used only by this benchmark) must agree on the same input.
byte[] reference;
using (var decompressor = new ZstdSharp.Decompressor())
reference = decompressor.Unwrap(input).ToArray();
Assert.Equal(expected, reference);

var elapsed = new List<TimeSpan>(timedRuns);
var referenceElapsed = new List<TimeSpan>(timedRuns);
for (int i = 0; i < timedRuns; i++)
{
// Alternate the order so that neither decoder always runs on the colder CPU.
if (i % 2 == 0)
{
elapsed.Add(TimeOurs(input, expected.Length));
referenceElapsed.Add(TimeReference(input, expected.Length));
}
else
{
referenceElapsed.Add(TimeReference(input, expected.Length));
elapsed.Add(TimeOurs(input, expected.Length));
}
}

TimeSpan best = elapsed.Min();
double seconds = best.TotalSeconds;
TimeSpan referenceBest = referenceElapsed.Min();
double referenceSeconds = referenceBest.TotalSeconds;
_output.WriteLine($"Input: {input.Length / (1024.0 * 1024):F1} MiB in {elapsed.Count} runs, content {expected.Length / (1024.0 * 1024):F1} MiB.");
_output.WriteLine($"Best decode, ours: {best.TotalMilliseconds:F0} ms, {input.Length / (1024.0 * 1024) / seconds:F1} MiB/s in, {expected.Length / (1024.0 * 1024) / seconds:F1} MiB/s out.");
_output.WriteLine($"Best decode, reference: {referenceBest.TotalMilliseconds:F0} ms, {input.Length / (1024.0 * 1024) / referenceSeconds:F1} MiB/s in, {expected.Length / (1024.0 * 1024) / referenceSeconds:F1} MiB/s out.");
}

private static TimeSpan TimeOurs(byte[] input, int expectedLength)
{
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, blocking: true);
var watch = Stopwatch.StartNew();
byte[] actual = ZstdDecoder.Decompress(input, expectedLength);
watch.Stop();
Assert.Equal(expectedLength, actual.Length);
return watch.Elapsed;
}

private static TimeSpan TimeReference(byte[] input, int expectedLength)
{
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, blocking: true);
var watch = Stopwatch.StartNew();
byte[] reference;
using (var decompressor = new ZstdSharp.Decompressor())
reference = decompressor.Unwrap(input).ToArray();
watch.Stop();
Assert.Equal(expectedLength, reference.Length);
return watch.Elapsed;
}
}
2 changes: 1 addition & 1 deletion tests/PclIO.Tests/ZstdDecoderTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ private static byte[] Content(string rule)

private static byte[] FrameOf(string name) => Convert.FromBase64String(ZstdVectors.All.Single(v => v.Name == name).Base64);

private static byte[] ContentOf(string name) => Content(ZstdVectors.All.Single(v => v.Name == name).Content);
internal static byte[] ContentOf(string name) => Content(ZstdVectors.All.Single(v => v.Name == name).Content);

// ---- Real frames --------------------------------------------------------------------------------------------------

Expand Down
Loading