From 6387ae7406b6a0f97a2ca22ed0ca54fb072ec65d Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:06:03 +0300 Subject: [PATCH 1/4] Expose the mode 2 chunk chain ReadChunks walks the chunk chain of an LZ block payload without decompressing it: each chunk reports its offset, header length, body size, output size and compression flag, and the chain is verified against the block's declared size (in real files the chunk output sizes add up to the block's unpacked size; a chain that does not add up is a scanner false positive and throws InvalidDataException). The compressed runs themselves are not decoded yet. --- CHANGELOG.md | 9 +++ docs/public-api.md | 2 + src/PlaIO/Internal/LzChunkWalker.cs | 35 +++++++++++ src/PlaIO/PlaDocument.cs | 36 +++++++++++ src/PlaIO/PlaLzChunk.cs | 25 ++++++++ tests/PlaIO.Tests/LzChunkTests.cs | 96 +++++++++++++++++++++++++++++ tests/PlaIO.Tests/Synthetic.cs | 24 ++++++++ 7 files changed, 227 insertions(+) create mode 100644 src/PlaIO/Internal/LzChunkWalker.cs create mode 100644 src/PlaIO/PlaLzChunk.cs create mode 100644 tests/PlaIO.Tests/LzChunkTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index dff1a25..d4fd603 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes are documented here. The format follows [Keep a Changelog](h and the project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html); versions before 1.0 may change the API between minor releases. +## [Unreleased] + +### Added + +- `PlaDocument.ReadChunks(PlaBlock)`: the chunk chain of a mode 2 (LZ) block payload without decompression. Each + `PlaLzChunk` reports `Offset`, `HeaderLength` (9 or 3), `PackedSize` (body), `UnpackedSize` and `IsCompressed`. + The chain is verified against the block's declared size; a chain that does not add up throws `InvalidDataException`. + The compressed runs themselves are not decoded yet. + ## [0.1.0] - 2026-11-12 ### Added diff --git a/docs/public-api.md b/docs/public-api.md index f3e6b94..b9edc8e 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -22,11 +22,13 @@ The `PlaIO` package is the supported surface listed here; every public type is i | `PlaDocument.Header` | The `FdbHeader`. | | `PlaDocument.Blocks` | The block index in file order, found by scanning from offset 32. No on-disk block map is known, so completeness is not claimed. The scan can take a byte run for a block header, so the index may contain false entries. In real files a small share of mode 1 blocks does not decode to its declared size, and `ReadPayload` throws `InvalidDataException` for them. | | `PlaDocument.ReadPayload(PlaBlock block)` | The decoded payload of a block of this document. Decodes mode 0 and mode 1. Throws `NotSupportedException` for mode 2 (LZ), `InvalidDataException` when a mode 1 payload does not decode to the declared size or the file is cut short, and `ArgumentException` when the block is not from this document. | +| `PlaDocument.ReadChunks(PlaBlock block)` | The chunk chain of a mode 2 (LZ) block, without decompression: each `PlaLzChunk` carries `Offset`, `HeaderLength` (9 or 3), `PackedSize` (body), `UnpackedSize` and `IsCompressed`. The chain is verified: the chunk headers must consume the payload exactly and the chunk output sizes must add up to the block's `UnpackedSize`. Throws `NotSupportedException` for a block of another mode, `InvalidDataException` when the chain does not add up (a scanner false positive in real files), and `ArgumentException` when the block is not from this document. The compressed runs themselves are not decoded yet. | | `PlaDocument.FindDirectories()` | Searches every raw (mode 0) block large enough for 64 records of 30 bytes for an object directory, at most one per block. Recomputed on each call; mode 1 payloads are not searched. | | `PlaDocument.Dispose()` | Releases the stream unless the document was opened with `leaveOpen`. | | `FdbHeader` | `Size` (64), `LayoutVersion` (u16 at offset 8), `FormatVersion` (u16 at offset 10), `Field12` (u32 at offset 12; meaning not documented), `Raw` (the first min(64, file length) bytes; bytes 32..63 overlap the first block). | | `PlaBlockMode` | `Raw` (0), `ZeroMask` (1, flag 0x100), `Lz` (2, flag 0x200; recognised, not decoded). | | `PlaBlock` | One indexed block: `Offset`, `Tag` (`BL` or `DY`), `Id` and `Kind` (raw), `Version` (0 or 1), `Flags` (raw), `SizeClass` (`Flags & 0xF`), `Mode`, `Guid` (present when flag 0x20 is set), `PackedSize`, `UnpackedSize`, `Extra` (present when flag 0x400 is set; raw), `PayloadOffset`, `HeaderLength`, `End`. | +| `PlaLzChunk` | One chunk of a mode 2 payload: `Offset` (within the payload), `HeaderLength` (9 or 3), `PackedSize` (the body after the header), `UnpackedSize` (its output size), `IsCompressed` (flags bit 0). In real files the chunk output sizes add up to the block's `UnpackedSize`. | | `ObjectDirectory` | A run of directory records inside a raw block: `Block`, `RecordOffset` (payload offset of the first record), `Phase` (`RecordOffset % 30`), `Records`, `AbuttingPairs`, `OverlappingPairs`, `DuplicatePositions`. The run is an estimate and can be shorter than the real directory: it stops after more than 8 consecutive records without an abutting neighbour. | | `ObjectDirectoryRecord` | The 30-byte record `[u16 class][u64 pos][u32 count][GUID16]` as `ClassNumber`, `Position`, `Count`, `ObjectId`. The units of position and count are not documented, and the class number is not mapped to a class name. | diff --git a/src/PlaIO/Internal/LzChunkWalker.cs b/src/PlaIO/Internal/LzChunkWalker.cs new file mode 100644 index 0000000..f68ea18 --- /dev/null +++ b/src/PlaIO/Internal/LzChunkWalker.cs @@ -0,0 +1,35 @@ +using System.Buffers.Binary; + +namespace PlaIO.Internal; + +/// Walks the chunk chain of a mode 2 (LZ) block payload: chunk headers only, no decompression. +internal static class LzChunkWalker +{ + public static List Walk(ReadOnlySpan payload, long blockOffset) + { + var chunks = new List(); + int pos = 0; + long sum = 0; + while (pos < payload.Length) + { + bool longHeader = (payload[pos] & 0x02) != 0; + int headerLength = longHeader ? 9 : 3; + if (pos + headerLength > payload.Length) + throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the chunk at payload offset {pos} has no room for its {headerLength}-byte header."); + long packed = longHeader + ? BinaryPrimitives.ReadUInt32LittleEndian(payload.Slice(pos + 1, 4)) + : payload[pos + 1]; + long unpacked = longHeader + ? BinaryPrimitives.ReadUInt32LittleEndian(payload.Slice(pos + 5, 4)) + : payload[pos + 2]; + if (packed < headerLength) + throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the chunk at payload offset {pos} declares {packed} packed bytes, less than its {headerLength}-byte header."); + if (pos + packed > payload.Length) + throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the chunk at payload offset {pos} runs past the payload by {pos + packed - payload.Length} bytes."); + sum += unpacked; + chunks.Add(new PlaLzChunk(pos, headerLength, packed - headerLength, unpacked, (payload[pos] & 0x01) != 0)); + pos += (int)packed; + } + return chunks; + } +} diff --git a/src/PlaIO/PlaDocument.cs b/src/PlaIO/PlaDocument.cs index 4cc7ad1..8692389 100644 --- a/src/PlaIO/PlaDocument.cs +++ b/src/PlaIO/PlaDocument.cs @@ -125,6 +125,42 @@ internal byte[] ReadPayload(PlaBlock block, out int consumed) } } + /// Walks the chunk chain of a mode 2 () block payload without decompressing + /// it: every chunk with its offset, header length, body size, output size and compression flag. + /// The compressed runs inside the chunks are not decoded (their algorithm is not documented yet), but the + /// chain itself is verified: the chunk headers must consume the payload exactly and the sum of the chunk output + /// sizes must equal the block's . Blocks whose chain does not add up are + /// reported as invalid, which in real files marks a scanner false positive. + /// The block does not use mode 2 (). + /// A chunk header does not fit, a chunk runs past the payload, or the chain + /// does not add up to the declared size. + /// The block is not one of the of this document. + public IReadOnlyList ReadChunks(PlaBlock block) + { + ArgumentNullException.ThrowIfNull(block); + ObjectDisposedException.ThrowIf(_disposed, this); + int index = _blocks.BinarySearch(block, ByOffset); + if (index < 0 || !ReferenceEquals(_blocks[index], block)) + throw new ArgumentException($"The block at 0x{block.Offset:x} is not in the block index of this document.", nameof(block)); + if (block.Mode != PlaBlockMode.Lz) + throw new NotSupportedException($"Block at 0x{block.Offset:x} uses payload mode {block.Mode}; only mode 2 (LZ) payloads are chunked."); + + var payload = new byte[block.PackedSize]; + StreamReads.ReadExactlyAt(_stream, block.PayloadOffset, payload, $"the payload of the block at 0x{block.Offset:x}"); + List chunks; + try + { + chunks = LzChunkWalker.Walk(payload, block.Offset); + } + catch (InvalidDataException e) + { + throw new InvalidDataException($"Block at 0x{block.Offset:x}: {e.Message}", e); + } + if (chunks.Sum(c => c.UnpackedSize) != block.UnpackedSize) + throw new InvalidDataException($"Mode-2 block at 0x{block.Offset:x}: the chunks produce {chunks.Sum(c => c.UnpackedSize)} bytes in total, the block declares {block.UnpackedSize}."); + return chunks.AsReadOnly(); + } + /// Searches every raw block large enough for 64 records for an object directory (at most one per block). /// Computed on each call; reads the raw blocks one at a time. Mode-1 payloads are not searched. /// The work grows with the total size of the raw blocks: each block gets a linear pre-check at all 30 record diff --git a/src/PlaIO/PlaLzChunk.cs b/src/PlaIO/PlaLzChunk.cs new file mode 100644 index 0000000..3cefcf3 --- /dev/null +++ b/src/PlaIO/PlaLzChunk.cs @@ -0,0 +1,25 @@ +namespace PlaIO; + +/// One chunk of a mode 2 (LZ) block payload: a self-contained run of output bytes with its own sizes, found +/// by . The payload is a chain of chunks; each chunk starts with a flags byte and +/// declares its sizes, so the chain can be walked without decoding the compressed runs. +/// +/// In real files whose chunk chain walks cleanly, the sum of over the chunks equals the +/// block's , and a chunk whose flags bit 0 is clear stores exactly +/// raw output bytes. A chain that does not add up, or a chunk header that does not fit, marks +/// a scanner false positive or an unknown detail; rejects such a block. +/// +/// Offset of the chunk within the block payload; the flags byte sits here. +/// Size of the chunk header: 9 bytes (flags, two 32-bit sizes) or 3 bytes (flags, two +/// 8-bit sizes). +/// Size of the chunk body after the header. A compressed chunk decodes these bytes to +/// ; a stored chunk ( false) holds them raw. +/// Number of output bytes the chunk produces. +/// True when the chunk body is a compressed run (flags bit 0 set); false when the body is +/// stored raw. +public readonly record struct PlaLzChunk( + long Offset, + int HeaderLength, + long PackedSize, + long UnpackedSize, + bool IsCompressed); diff --git a/tests/PlaIO.Tests/LzChunkTests.cs b/tests/PlaIO.Tests/LzChunkTests.cs new file mode 100644 index 0000000..9b42981 --- /dev/null +++ b/tests/PlaIO.Tests/LzChunkTests.cs @@ -0,0 +1,96 @@ +using Xunit; + +namespace PlaIO.Tests; + +/// The chunk chain of mode-2 (LZ) blocks, read by on synthetic payloads: +/// stored and compressed chunks, short and long headers, and the invalid chains the reader must reject. +public class LzChunkTests +{ + [Fact] + public void ChainOfStoredAndCompressedChunks() + { + var body1 = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + var body2 = new byte[] { 11, 12, 13, 14 }; + var payload = Synthetic.LzPayload( + [ + Synthetic.LzChunk(compressed: false, longHeader: true, bodySize: body1.Length, unpacked: body1.Length, body1), + Synthetic.LzChunk(compressed: true, longHeader: true, bodySize: body2.Length, unpacked: 40, body2), + Synthetic.LzChunk(compressed: false, longHeader: false, bodySize: 6, unpacked: 6), + ]); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.Block(PlaBlockMode.Lz, payload, unpacked: 56)))); + + var chunks = document.ReadChunks(document.Blocks.Single()); + + Assert.Equal(3, chunks.Count); + Assert.Equal(0, chunks[0].Offset); + Assert.Equal(9, chunks[0].HeaderLength); + Assert.Equal(body1.Length, chunks[0].PackedSize); + Assert.Equal(body1.Length, chunks[0].UnpackedSize); + Assert.False(chunks[0].IsCompressed); + Assert.Equal(9 + body1.Length, chunks[1].Offset); + Assert.True(chunks[1].IsCompressed); + Assert.Equal(body2.Length, chunks[1].PackedSize); + Assert.Equal(40, chunks[1].UnpackedSize); + Assert.Equal(9 + body1.Length + 9 + body2.Length, chunks[2].Offset); + Assert.Equal(3, chunks[2].HeaderLength); + Assert.Equal(6, chunks[2].PackedSize); + Assert.Equal(6, chunks[2].UnpackedSize); + Assert.False(chunks[2].IsCompressed); + } + + [Fact] + public void AChainWithTrailingPaddingIsRejected() + { + var payload = Synthetic.LzPayload( + [ + Synthetic.LzChunk(compressed: false, longHeader: true, bodySize: 10, unpacked: 10), + ], tail: 4); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.Block(PlaBlockMode.Lz, payload, unpacked: 10)))); + + Assert.Throws(() => document.ReadChunks(document.Blocks.Single())); + } + + [Fact] + public void AChainThatDoesNotAddUpIsRejected() + { + var payload = Synthetic.LzPayload( + [ + Synthetic.LzChunk(compressed: false, longHeader: true, bodySize: 10, unpacked: 10), + Synthetic.LzChunk(compressed: false, longHeader: true, bodySize: 4, unpacked: 4), + ]); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.Block(PlaBlockMode.Lz, payload, unpacked: 20)))); + + Assert.Throws(() => document.ReadChunks(document.Blocks.Single())); + } + + [Fact] + public void AChunkThatRunsPastThePayloadIsRejected() + { + var payload = Synthetic.LzPayload( + [ + Synthetic.LzChunk(compressed: false, longHeader: true, bodySize: 10, unpacked: 10), + ]); + // Cut the payload short: the header says 19 bytes, only 19 - 4 remain. + payload = payload[..^4]; + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.Block(PlaBlockMode.Lz, payload, unpacked: 10)))); + + Assert.Throws(() => document.ReadChunks(document.Blocks.Single())); + } + + [Fact] + public void ABlockOfAnotherModeIsRejected() + { + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.Block(PlaBlockMode.Raw, [1, 2, 3])))); + + Assert.Throws(() => document.ReadChunks(document.Blocks.Single())); + } + + [Fact] + public void ABlockOutsideTheIndexIsRejected() + { + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.Block(PlaBlockMode.Lz, Synthetic.LzPayload([Synthetic.LzChunk(compressed: false, longHeader: true, bodySize: 10, unpacked: 10)]), unpacked: 10)))); + + var stranger = new PlaBlock(0x1234, "BL", 1, 0xFDEA, 0, 0x8080, PlaBlockMode.Lz, null, 10, 10, null, document.Blocks.Single().PayloadOffset); + Assert.Throws(() => document.ReadChunks(stranger)); + } +} diff --git a/tests/PlaIO.Tests/Synthetic.cs b/tests/PlaIO.Tests/Synthetic.cs index 75979fd..94c1ae4 100644 --- a/tests/PlaIO.Tests/Synthetic.cs +++ b/tests/PlaIO.Tests/Synthetic.cs @@ -125,6 +125,30 @@ public static byte[] RandomBytes(Random random, int length) /// A container: the 32-byte header, then the parts from offset 32. public static byte[] Fdb(params byte[][] parts) => Header().Concat(parts.SelectMany(p => p)).ToArray(); + /// A mode-2 chunk header followed by . Long header (9 bytes) when + /// ; bit 0 of the flags marks the body compressed. + public static byte[] LzChunk(bool compressed, bool longHeader, long bodySize, long unpacked, byte[]? body = null) + { + byte flags = (byte)((compressed ? 0x01 : 0x00) | (longHeader ? 0x02 : 0x00) | 0x44); + var bytes = new List { flags }; + if (longHeader) + { + bytes.AddRange(Le32((uint)bodySize + 9)); + bytes.AddRange(Le32((uint)unpacked)); + } + else + { + if (bodySize + 3 > 255 || unpacked > 255) throw new ArgumentOutOfRangeException(nameof(bodySize), "A short-header chunk holds at most 252 body bytes."); + bytes.Add((byte)(bodySize + 3)); + bytes.Add((byte)unpacked); + } + bytes.AddRange(body ?? new byte[bodySize]); + return bytes.ToArray(); + } + + /// An LZ payload: the chunk chain from the parts, then zero bytes. + public static byte[] LzPayload(byte[][] chunks, int tail = 0) => Concat(chunks).Concat(new byte[tail]).ToArray(); + /// A PNG: the signature and each chunk as [BE length][type][data][CRC]; the CRC is zero (not checked). public static byte[] Png(params (string Type, byte[] Data)[] chunks) { From 199274851a58e98f562366819efc08d33f6e1f34 Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:52:00 +0300 Subject: [PATCH 2/4] Decode mode 2 (LZ) payloads in ReadPayload The chunk-chain model was an artifact of the walk: what looked like internal chunk headers is the encoder writing its own sizes as the first literals of one token stream. ReadPayload now decodes the real mode 2 format: a token byte (high nibble = literal count, 15 extended by 255-chained bytes), the literals, a big-endian 16-bit distance and a length byte (0 means 256, otherwise the token's low nibble is the high byte of the length). Distance 0 emits zeros; a positive distance copies from that many bytes back in the global window, byte by byte, so overlapping copies replicate a repeating pattern. The stream produces exactly the block's declared unpacked size; the body tail is writer slack. A mode 2 token stream may reference the output of any preceding block, so ReadPayload decodes the file from the start into one global window (raw payloads, full zero-mask decodes, mode-2 outputs, in block order) and returns the requested block's output; the cost of the call grows with the size of the file. Blocks that do not decode (false entries of the scan) are skipped and rolled back out of the window; the cross-check against the object directories confirmed the resulting window matches the real stream. ReadChunks, PlaLzChunk and LzChunkWalker are removed. On the six controlled saves every mode 2 block (2432 total) decodes to its declared size; the sample test now asserts that instead of the old NotSupportedException. --- CHANGELOG.md | 15 +++- docs/public-api.md | 6 +- src/PlaIO/Internal/LzChunkWalker.cs | 35 -------- src/PlaIO/Internal/LzPayloadDecoder.cs | 82 +++++++++++++++++ src/PlaIO/PlaDocument.cs | 119 ++++++++++++++++++------- src/PlaIO/PlaLzChunk.cs | 25 ------ tests/PlaIO.Tests/LzChunkTests.cs | 96 -------------------- tests/PlaIO.Tests/LzPayloadTests.cs | 110 +++++++++++++++++++++++ tests/PlaIO.Tests/PlaDocumentTests.cs | 7 +- tests/PlaIO.Tests/SampleTests.cs | 15 ++-- 10 files changed, 306 insertions(+), 204 deletions(-) delete mode 100644 src/PlaIO/Internal/LzChunkWalker.cs create mode 100644 src/PlaIO/Internal/LzPayloadDecoder.cs delete mode 100644 src/PlaIO/PlaLzChunk.cs delete mode 100644 tests/PlaIO.Tests/LzChunkTests.cs create mode 100644 tests/PlaIO.Tests/LzPayloadTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index d4fd603..045a45c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,17 @@ API between minor releases. ### Added -- `PlaDocument.ReadChunks(PlaBlock)`: the chunk chain of a mode 2 (LZ) block payload without decompression. Each - `PlaLzChunk` reports `Offset`, `HeaderLength` (9 or 3), `PackedSize` (body), `UnpackedSize` and `IsCompressed`. - The chain is verified against the block's declared size; a chain that does not add up throws `InvalidDataException`. - The compressed runs themselves are not decoded yet. +- `PlaDocument.ReadPayload` decodes mode 2 (LZ) payloads. The mode 2 token stream produces exactly the block's declared + unpacked size and may reference the decompressed output of any preceding block of the file, so `ReadPayload` for a + mode 2 block decodes the file from the start into one global window; the cost of the call grows with the size of the + file. Preceding blocks that do not decode (false entries of the scan) are skipped and leave the window unchanged. A + stream that does not produce the declared size, or a back-reference beyond the decoded stream, throws + `InvalidDataException`. + +### Changed + +- `PlaDocument.ReadPayload` no longer throws `NotSupportedException` for mode 2 blocks: all three storage modes + (raw, zero mask, LZ) decode now. ## [0.1.0] - 2026-11-12 diff --git a/docs/public-api.md b/docs/public-api.md index b9edc8e..6b94506 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -21,14 +21,12 @@ The `PlaIO` package is the supported surface listed here; every public type is i | `PlaDocument.Length` | Length of the file in bytes. | | `PlaDocument.Header` | The `FdbHeader`. | | `PlaDocument.Blocks` | The block index in file order, found by scanning from offset 32. No on-disk block map is known, so completeness is not claimed. The scan can take a byte run for a block header, so the index may contain false entries. In real files a small share of mode 1 blocks does not decode to its declared size, and `ReadPayload` throws `InvalidDataException` for them. | -| `PlaDocument.ReadPayload(PlaBlock block)` | The decoded payload of a block of this document. Decodes mode 0 and mode 1. Throws `NotSupportedException` for mode 2 (LZ), `InvalidDataException` when a mode 1 payload does not decode to the declared size or the file is cut short, and `ArgumentException` when the block is not from this document. | -| `PlaDocument.ReadChunks(PlaBlock block)` | The chunk chain of a mode 2 (LZ) block, without decompression: each `PlaLzChunk` carries `Offset`, `HeaderLength` (9 or 3), `PackedSize` (body), `UnpackedSize` and `IsCompressed`. The chain is verified: the chunk headers must consume the payload exactly and the chunk output sizes must add up to the block's `UnpackedSize`. Throws `NotSupportedException` for a block of another mode, `InvalidDataException` when the chain does not add up (a scanner false positive in real files), and `ArgumentException` when the block is not from this document. The compressed runs themselves are not decoded yet. | +| `PlaDocument.ReadPayload(PlaBlock block)` | The decoded payload of a block of this document. Decodes all three storage modes: raw (mode 0), zero masks (mode 1) and the LZ token stream (mode 2). A mode 2 payload is produced by decoding every preceding block of the file in order into one global window (the token stream may reference the output of any preceding block), so the cost of the call grows with the size of the file; preceding blocks that do not decode (false entries of the scan) are skipped and leave the window unchanged. Throws `InvalidDataException` when a payload does not decode to the declared size or the file is cut short, and `ArgumentException` when the block is not from this document. | | `PlaDocument.FindDirectories()` | Searches every raw (mode 0) block large enough for 64 records of 30 bytes for an object directory, at most one per block. Recomputed on each call; mode 1 payloads are not searched. | | `PlaDocument.Dispose()` | Releases the stream unless the document was opened with `leaveOpen`. | | `FdbHeader` | `Size` (64), `LayoutVersion` (u16 at offset 8), `FormatVersion` (u16 at offset 10), `Field12` (u32 at offset 12; meaning not documented), `Raw` (the first min(64, file length) bytes; bytes 32..63 overlap the first block). | -| `PlaBlockMode` | `Raw` (0), `ZeroMask` (1, flag 0x100), `Lz` (2, flag 0x200; recognised, not decoded). | +| `PlaBlockMode` | `Raw` (0), `ZeroMask` (1, flag 0x100), `Lz` (2, flag 0x200). | | `PlaBlock` | One indexed block: `Offset`, `Tag` (`BL` or `DY`), `Id` and `Kind` (raw), `Version` (0 or 1), `Flags` (raw), `SizeClass` (`Flags & 0xF`), `Mode`, `Guid` (present when flag 0x20 is set), `PackedSize`, `UnpackedSize`, `Extra` (present when flag 0x400 is set; raw), `PayloadOffset`, `HeaderLength`, `End`. | -| `PlaLzChunk` | One chunk of a mode 2 payload: `Offset` (within the payload), `HeaderLength` (9 or 3), `PackedSize` (the body after the header), `UnpackedSize` (its output size), `IsCompressed` (flags bit 0). In real files the chunk output sizes add up to the block's `UnpackedSize`. | | `ObjectDirectory` | A run of directory records inside a raw block: `Block`, `RecordOffset` (payload offset of the first record), `Phase` (`RecordOffset % 30`), `Records`, `AbuttingPairs`, `OverlappingPairs`, `DuplicatePositions`. The run is an estimate and can be shorter than the real directory: it stops after more than 8 consecutive records without an abutting neighbour. | | `ObjectDirectoryRecord` | The 30-byte record `[u16 class][u64 pos][u32 count][GUID16]` as `ClassNumber`, `Position`, `Count`, `ObjectId`. The units of position and count are not documented, and the class number is not mapped to a class name. | diff --git a/src/PlaIO/Internal/LzChunkWalker.cs b/src/PlaIO/Internal/LzChunkWalker.cs deleted file mode 100644 index f68ea18..0000000 --- a/src/PlaIO/Internal/LzChunkWalker.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System.Buffers.Binary; - -namespace PlaIO.Internal; - -/// Walks the chunk chain of a mode 2 (LZ) block payload: chunk headers only, no decompression. -internal static class LzChunkWalker -{ - public static List Walk(ReadOnlySpan payload, long blockOffset) - { - var chunks = new List(); - int pos = 0; - long sum = 0; - while (pos < payload.Length) - { - bool longHeader = (payload[pos] & 0x02) != 0; - int headerLength = longHeader ? 9 : 3; - if (pos + headerLength > payload.Length) - throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the chunk at payload offset {pos} has no room for its {headerLength}-byte header."); - long packed = longHeader - ? BinaryPrimitives.ReadUInt32LittleEndian(payload.Slice(pos + 1, 4)) - : payload[pos + 1]; - long unpacked = longHeader - ? BinaryPrimitives.ReadUInt32LittleEndian(payload.Slice(pos + 5, 4)) - : payload[pos + 2]; - if (packed < headerLength) - throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the chunk at payload offset {pos} declares {packed} packed bytes, less than its {headerLength}-byte header."); - if (pos + packed > payload.Length) - throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the chunk at payload offset {pos} runs past the payload by {pos + packed - payload.Length} bytes."); - sum += unpacked; - chunks.Add(new PlaLzChunk(pos, headerLength, packed - headerLength, unpacked, (payload[pos] & 0x01) != 0)); - pos += (int)packed; - } - return chunks; - } -} diff --git a/src/PlaIO/Internal/LzPayloadDecoder.cs b/src/PlaIO/Internal/LzPayloadDecoder.cs new file mode 100644 index 0000000..0021fff --- /dev/null +++ b/src/PlaIO/Internal/LzPayloadDecoder.cs @@ -0,0 +1,82 @@ +using System.Buffers.Binary; + +namespace PlaIO.Internal; + +/// Decodes the mode 2 (LZ) token stream of one block payload into the +/// block output (exactly the block's declared unpacked size), appending every +/// produced byte to the window — the global decompressed stream of the whole +/// file. Back-references reach anywhere inside that stream, including the output of +/// preceding blocks. +/// +/// The stream is a chain of sequences. A sequence starts with a token byte: the high nibble +/// is the number of literal bytes that follow (15 = extended: the following bytes are added while +/// they equal 255). After the literals, if the output is not complete yet, come the match fields: +/// a 16-bit big-endian distance and a length byte. A distance of 0 emits the match length as zero +/// bytes (RLE); a positive distance copies the match length from that many bytes back in the +/// global window, byte by byte, so overlapping copies replicate a repeating pattern. The low +/// nibble of the token is the high byte of the match length, so a match spans up to 4095 bytes +/// (a length byte of 0 with a non-zero high nibble still means 256). When the output is full the +/// sequence ends; the remaining bytes of the body are slack the writer reserved and are not +/// consumed. +/// +internal static class LzPayloadDecoder +{ + public static int Decode(ReadOnlySpan body, Span output, List window, long blockOffset) + { + int src = 0, o = 0; + while (o < output.Length) + { + if (src >= body.Length) + throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the token stream ends after {o} of {output.Length} output bytes."); + int token = body[src++]; + int litLen = token >> 4; + if (litLen == 15) + { + byte x; + do + { + if (src >= body.Length) + throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the literal length extension runs past the payload."); + x = body[src++]; + litLen += x; + } + while (x == 255 && src < body.Length); + } + if (src + litLen > body.Length) + throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the literal run of {litLen} bytes runs past the payload at {src}."); + int litEnd = Math.Min(o + litLen, output.Length); + for (; o < litEnd; o++) + { + byte v = body[src++]; + output[o] = v; + window.Add(v); + } + if (o >= output.Length) + return src; // trailing literals completed the block; the body tail is slack + if (src + 2 > body.Length) + throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the distance of the sequence at {src - 1} runs past the payload."); + int dist = (body[src] << 8) | body[src + 1]; + src += 2; + if (src >= body.Length) + throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the match length of the sequence at {src - 2} runs past the payload."); + int lenByte = body[src++]; + int matchLen = lenByte == 0 ? 256 : (((token & 0xF) << 8) | lenByte); + if (o + matchLen > output.Length) + matchLen = output.Length - o; // the final match is clipped at the declared size + if (dist == 0) + { + for (int i = 0; i < matchLen; i++, o++) { output[o] = 0; window.Add(0); } + continue; + } + if (dist > window.Count) + throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the back-reference distance {dist} exceeds the decoded stream ({window.Count} bytes)."); + for (int i = 0; i < matchLen; i++, o++) + { + byte v = window[window.Count - dist]; + output[o] = v; + window.Add(v); + } + } + return src; + } +} diff --git a/src/PlaIO/PlaDocument.cs b/src/PlaIO/PlaDocument.cs index 8692389..52997d1 100644 --- a/src/PlaIO/PlaDocument.cs +++ b/src/PlaIO/PlaDocument.cs @@ -76,8 +76,11 @@ public static PlaDocument Open(Stream stream, bool leaveOpen = false) } /// The decoded payload of a block of this document. - /// The block uses mode 2 (). - /// A mode-1 stream does not decode to the declared size, or the file is cut + /// Reading a mode 2 () block decodes every block of the file in order into + /// one global window first, because a mode 2 token stream may reference the output of any preceding block; the cost + /// of the call therefore grows with the size of the file. A preceding block that does not decode to its declared + /// size (a false entry of the scan) is skipped. + /// A payload does not decode to its declared size, or the file is cut /// short. /// The block is not one of the of this document. public byte[] ReadPayload(PlaBlock block) => ReadPayload(block, out _); @@ -120,45 +123,101 @@ internal byte[] ReadPayload(PlaBlock block, out int consumed) } return payload; } + case PlaBlockMode.Lz: + { + // The mode-2 token stream references the decompressed output of all preceding blocks, so the whole + // file is decoded from the start and the requested block's output is returned. The cost of a call + // therefore grows with the size of the file. A preceding block that does not decode is a scanner + // false positive and not part of the real stream, so it is skipped. + var window = new List(); + foreach (var b in _blocks) + { + if (ReferenceEquals(b, block)) break; + TryAppendToWindow(b, window); + } + var blockPayload = new byte[block.PackedSize]; + StreamReads.ReadExactlyAt(_stream, block.PayloadOffset, blockPayload, $"the payload of the block at 0x{block.Offset:x}"); + // Mode 2 (LZ): the token stream produces exactly the declared unpacked size and may reference the + // window; the body tail after the output is complete is writer slack. + var output = new byte[block.UnpackedSize]; + try + { + consumed = LzPayloadDecoder.Decode(blockPayload, output, window, block.Offset); + } + catch (InvalidDataException e) + { + throw new InvalidDataException($"Block at 0x{block.Offset:x}: {e.Message}", e); + } + return output; + } default: - throw new NotSupportedException($"Block at 0x{block.Offset:x} uses payload mode 2 (LZ). Its compression is not documented yet; only modes 0 and 1 can be decoded."); + throw new NotSupportedException($"Block at 0x{block.Offset:x} uses an unknown payload mode {block.Mode}."); } } - /// Walks the chunk chain of a mode 2 () block payload without decompressing - /// it: every chunk with its offset, header length, body size, output size and compression flag. - /// The compressed runs inside the chunks are not decoded (their algorithm is not documented yet), but the - /// chain itself is verified: the chunk headers must consume the payload exactly and the sum of the chunk output - /// sizes must equal the block's . Blocks whose chain does not add up are - /// reported as invalid, which in real files marks a scanner false positive. - /// The block does not use mode 2 (). - /// A chunk header does not fit, a chunk runs past the payload, or the chain - /// does not add up to the declared size. - /// The block is not one of the of this document. - public IReadOnlyList ReadChunks(PlaBlock block) + /// Appends the decoded payload of one block to the mode-2 global window, or leaves the window unchanged when + /// the block does not decode to its declared size (a false entry of the scan is not part of the real stream, so a + /// partial decode is rolled back). Returns whether the block was appended. + private bool TryAppendToWindow(PlaBlock b, List window) + { + int before = window.Count; + try + { + AppendToWindow(b, window); + return true; + } + catch (InvalidDataException) + { + window.RemoveRange(before, window.Count - before); + return false; + } + } + + /// Decodes the whole file once into the global window: the raw payloads, the full zero-mask decodes and the + /// mode-2 outputs, in block order. Blocks that do not decode to their declared size (false entries of the scan) are + /// skipped and reported in ; the window stays as if they were not there. + internal List DecodeGlobalWindow(out IReadOnlyList skipped) { - ArgumentNullException.ThrowIfNull(block); ObjectDisposedException.ThrowIf(_disposed, this); - int index = _blocks.BinarySearch(block, ByOffset); - if (index < 0 || !ReferenceEquals(_blocks[index], block)) - throw new ArgumentException($"The block at 0x{block.Offset:x} is not in the block index of this document.", nameof(block)); - if (block.Mode != PlaBlockMode.Lz) - throw new NotSupportedException($"Block at 0x{block.Offset:x} uses payload mode {block.Mode}; only mode 2 (LZ) payloads are chunked."); + var window = new List(); + var failed = new List(); + foreach (var b in _blocks) + { + if (!TryAppendToWindow(b, window)) failed.Add(b); + } + skipped = failed.AsReadOnly(); + return window; + } - var payload = new byte[block.PackedSize]; - StreamReads.ReadExactlyAt(_stream, block.PayloadOffset, payload, $"the payload of the block at 0x{block.Offset:x}"); - List chunks; - try + /// Appends the decoded payload of one block to the mode-2 global window. Raw payloads are copied as they + /// are; a zero-mask payload contributes its full decode; a mode 2 payload is decoded and the decoder appends every + /// produced byte. Throws when the block does not decode to its declared size. + private void AppendToWindow(PlaBlock b, List window) + { + var payload = new byte[b.PackedSize]; + StreamReads.ReadExactlyAt(_stream, b.PayloadOffset, payload, $"the payload of the block at 0x{b.Offset:x}"); + if (b.Mode == PlaBlockMode.Raw) { - chunks = LzChunkWalker.Walk(payload, block.Offset); + window.AddRange(payload); + return; } - catch (InvalidDataException e) + if (b.Mode == PlaBlockMode.ZeroMask) { - throw new InvalidDataException($"Block at 0x{block.Offset:x}: {e.Message}", e); + // A mask byte yields at most 8 bytes, so a larger declared size cannot decode; reject it before allocating. + if ((b.UnpackedSize + 7) / 8 > b.PackedSize) + throw new InvalidDataException($"Mode-1 block at 0x{b.Offset:x} declares {b.UnpackedSize} unpacked bytes from {b.PackedSize} packed bytes; a mask byte yields at most 8."); + long needed = Math.Min(b.PackedSize, b.UnpackedSize + (b.UnpackedSize + 7) / 8); + var source = new byte[needed]; + StreamReads.ReadExactlyAt(_stream, b.PayloadOffset, source, $"the payload of the block at 0x{b.Offset:x}"); + var decoded = new byte[b.UnpackedSize]; + PayloadDecoder.DecodeZeroMask(source, decoded); + window.AddRange(decoded); + return; } - if (chunks.Sum(c => c.UnpackedSize) != block.UnpackedSize) - throw new InvalidDataException($"Mode-2 block at 0x{block.Offset:x}: the chunks produce {chunks.Sum(c => c.UnpackedSize)} bytes in total, the block declares {block.UnpackedSize}."); - return chunks.AsReadOnly(); + // Mode 2 (LZ): the token stream produces exactly the declared unpacked size and the decoder appends + // every produced byte to the window; the body tail after the output is complete is writer slack. + var output = new byte[b.UnpackedSize]; + LzPayloadDecoder.Decode(payload, output, window, b.Offset); } /// Searches every raw block large enough for 64 records for an object directory (at most one per block). diff --git a/src/PlaIO/PlaLzChunk.cs b/src/PlaIO/PlaLzChunk.cs deleted file mode 100644 index 3cefcf3..0000000 --- a/src/PlaIO/PlaLzChunk.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace PlaIO; - -/// One chunk of a mode 2 (LZ) block payload: a self-contained run of output bytes with its own sizes, found -/// by . The payload is a chain of chunks; each chunk starts with a flags byte and -/// declares its sizes, so the chain can be walked without decoding the compressed runs. -/// -/// In real files whose chunk chain walks cleanly, the sum of over the chunks equals the -/// block's , and a chunk whose flags bit 0 is clear stores exactly -/// raw output bytes. A chain that does not add up, or a chunk header that does not fit, marks -/// a scanner false positive or an unknown detail; rejects such a block. -/// -/// Offset of the chunk within the block payload; the flags byte sits here. -/// Size of the chunk header: 9 bytes (flags, two 32-bit sizes) or 3 bytes (flags, two -/// 8-bit sizes). -/// Size of the chunk body after the header. A compressed chunk decodes these bytes to -/// ; a stored chunk ( false) holds them raw. -/// Number of output bytes the chunk produces. -/// True when the chunk body is a compressed run (flags bit 0 set); false when the body is -/// stored raw. -public readonly record struct PlaLzChunk( - long Offset, - int HeaderLength, - long PackedSize, - long UnpackedSize, - bool IsCompressed); diff --git a/tests/PlaIO.Tests/LzChunkTests.cs b/tests/PlaIO.Tests/LzChunkTests.cs deleted file mode 100644 index 9b42981..0000000 --- a/tests/PlaIO.Tests/LzChunkTests.cs +++ /dev/null @@ -1,96 +0,0 @@ -using Xunit; - -namespace PlaIO.Tests; - -/// The chunk chain of mode-2 (LZ) blocks, read by on synthetic payloads: -/// stored and compressed chunks, short and long headers, and the invalid chains the reader must reject. -public class LzChunkTests -{ - [Fact] - public void ChainOfStoredAndCompressedChunks() - { - var body1 = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; - var body2 = new byte[] { 11, 12, 13, 14 }; - var payload = Synthetic.LzPayload( - [ - Synthetic.LzChunk(compressed: false, longHeader: true, bodySize: body1.Length, unpacked: body1.Length, body1), - Synthetic.LzChunk(compressed: true, longHeader: true, bodySize: body2.Length, unpacked: 40, body2), - Synthetic.LzChunk(compressed: false, longHeader: false, bodySize: 6, unpacked: 6), - ]); - using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.Block(PlaBlockMode.Lz, payload, unpacked: 56)))); - - var chunks = document.ReadChunks(document.Blocks.Single()); - - Assert.Equal(3, chunks.Count); - Assert.Equal(0, chunks[0].Offset); - Assert.Equal(9, chunks[0].HeaderLength); - Assert.Equal(body1.Length, chunks[0].PackedSize); - Assert.Equal(body1.Length, chunks[0].UnpackedSize); - Assert.False(chunks[0].IsCompressed); - Assert.Equal(9 + body1.Length, chunks[1].Offset); - Assert.True(chunks[1].IsCompressed); - Assert.Equal(body2.Length, chunks[1].PackedSize); - Assert.Equal(40, chunks[1].UnpackedSize); - Assert.Equal(9 + body1.Length + 9 + body2.Length, chunks[2].Offset); - Assert.Equal(3, chunks[2].HeaderLength); - Assert.Equal(6, chunks[2].PackedSize); - Assert.Equal(6, chunks[2].UnpackedSize); - Assert.False(chunks[2].IsCompressed); - } - - [Fact] - public void AChainWithTrailingPaddingIsRejected() - { - var payload = Synthetic.LzPayload( - [ - Synthetic.LzChunk(compressed: false, longHeader: true, bodySize: 10, unpacked: 10), - ], tail: 4); - using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.Block(PlaBlockMode.Lz, payload, unpacked: 10)))); - - Assert.Throws(() => document.ReadChunks(document.Blocks.Single())); - } - - [Fact] - public void AChainThatDoesNotAddUpIsRejected() - { - var payload = Synthetic.LzPayload( - [ - Synthetic.LzChunk(compressed: false, longHeader: true, bodySize: 10, unpacked: 10), - Synthetic.LzChunk(compressed: false, longHeader: true, bodySize: 4, unpacked: 4), - ]); - using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.Block(PlaBlockMode.Lz, payload, unpacked: 20)))); - - Assert.Throws(() => document.ReadChunks(document.Blocks.Single())); - } - - [Fact] - public void AChunkThatRunsPastThePayloadIsRejected() - { - var payload = Synthetic.LzPayload( - [ - Synthetic.LzChunk(compressed: false, longHeader: true, bodySize: 10, unpacked: 10), - ]); - // Cut the payload short: the header says 19 bytes, only 19 - 4 remain. - payload = payload[..^4]; - using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.Block(PlaBlockMode.Lz, payload, unpacked: 10)))); - - Assert.Throws(() => document.ReadChunks(document.Blocks.Single())); - } - - [Fact] - public void ABlockOfAnotherModeIsRejected() - { - using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.Block(PlaBlockMode.Raw, [1, 2, 3])))); - - Assert.Throws(() => document.ReadChunks(document.Blocks.Single())); - } - - [Fact] - public void ABlockOutsideTheIndexIsRejected() - { - using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb(Synthetic.Block(PlaBlockMode.Lz, Synthetic.LzPayload([Synthetic.LzChunk(compressed: false, longHeader: true, bodySize: 10, unpacked: 10)]), unpacked: 10)))); - - var stranger = new PlaBlock(0x1234, "BL", 1, 0xFDEA, 0, 0x8080, PlaBlockMode.Lz, null, 10, 10, null, document.Blocks.Single().PayloadOffset); - Assert.Throws(() => document.ReadChunks(stranger)); - } -} diff --git a/tests/PlaIO.Tests/LzPayloadTests.cs b/tests/PlaIO.Tests/LzPayloadTests.cs new file mode 100644 index 0000000..f0ee5ab --- /dev/null +++ b/tests/PlaIO.Tests/LzPayloadTests.cs @@ -0,0 +1,110 @@ +using System.Buffers.Binary; +using Xunit; + +namespace PlaIO.Tests; + +/// The mode 2 (LZ) payload decode on synthetic containers: literal runs, back-references across +/// blocks (the global window), overlapping copies, RLE zeros, the length byte conventions and the +/// trailing slack. The byte sequences are the model verified against real files (see the format spec). +public class LzPayloadTests +{ + [Fact] + public void LiteralsAndBackReferencesAcrossBlocks() + { + // Block A: raw, provides the window. Block B: mode 2, references it. + byte[] a = "ABCDEFGH"u8.ToArray(); + // seq1: token 0x40 (lit 4) "wxyz"; dist 8 -> copies "EFG" (source at 12-8=4) + // seq2: token 0x00 (lit 0); dist 0; len 4 -> four zeros (window is now 19 bytes) + // seq3: token 0x00 (lit 0); dist 15 -> copies "EFGH" (source at 19-15=4, block A's bytes: + // the back-reference reaches across the block boundary into the raw block) + // trailing slack: FF FE + var lz = Synthetic.Concat( + [0x40], "wxyz"u8.ToArray(), [0x00, 0x08], [0x03], + [0x00], [0x00, 0x00], [0x04], + [0x00], [0x00, 0x0F], [0x04], + [0xFF, 0xFE]); + byte[] expected = "wxyzEFG\0\0\0\0EFGH"u8.ToArray(); + + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Synthetic.Block(PlaBlockMode.Raw, a), + Synthetic.Block(PlaBlockMode.Lz, lz, unpacked: (uint)expected.Length)))); + + var payload = document.ReadPayload(document.Blocks[1]); + Assert.Equal(expected, payload); + } + + [Fact] + public void LengthByteZeroMeans256AndHighNibbleExtendsTheLength() + { + // seq1: token 0x40 (lit 4) "ABCD"; dist 4, lenByte 4 -> len 4: copies the literals back, + // giving "ABCDABCD". seq2: token 0x01 (lit 0, lenHi 1): dist 4, lenByte 0 -> len 256; the + // overlapping copy repeats "ABCD" 64 times. seq3: token 0x10 (lit 1) "Z". + var lz = Synthetic.Concat( + [0x40], "ABCD"u8.ToArray(), [0x00, 0x04], [0x04], + [0x01], [0x00, 0x04], [0x00], + [0x10], "Z"u8.ToArray()); + byte[] expected = "ABCDABCD"u8.ToArray(); + for (int i = 0; i < 64; i++) expected = expected.Concat("ABCD"u8.ToArray()).ToArray(); + expected = expected.Concat("Z"u8.ToArray()).ToArray(); + + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Synthetic.Block(PlaBlockMode.Lz, lz, unpacked: (uint)expected.Length)))); + + var payload = document.ReadPayload(document.Blocks.Single()); + Assert.Equal(expected, payload); + } + + [Fact] + public void OverlappingCopyReplicatesARepeatingPattern() + { + // window tail "ABCD"; token 0x01 (lit 0, lenHi 1): dist 4, lenByte 45 -> len 0x12D = 301: + // the 4-byte pattern repeats with a 1-byte overlap offset... the copy replicates the + // pattern; the output length is clipped at the declared size. + var lz = Synthetic.Concat( + [0x40], "ABCD"u8.ToArray(), [0x00, 0x04], [0x04], + [0x01], [0x00, 0x04], [0x2D]); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Synthetic.Block(PlaBlockMode.Lz, lz, unpacked: 301)))); + + var payload = document.ReadPayload(document.Blocks.Single()); + Assert.Equal(301, payload.Length); + for (int i = 0; i < 301; i++) Assert.Equal("ABCD"[i % 4], (char)payload[i]); + } + + [Fact] + public void TrailingLiteralsEndTheBlockAndBodyTailIsSlack() + { + // token 0xF5: litLen 15 + ext byte 5 = 20 literals; no match follows; the body tail (2 bytes) is slack. + var lz = Synthetic.Concat( + [0xF5, 0x05], "ABCDEFGHIJKLMNOPQRST"u8.ToArray(), + [0xDE, 0xAD]); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Synthetic.Block(PlaBlockMode.Lz, lz, unpacked: 20)))); + + var payload = document.ReadPayload(document.Blocks.Single()); + Assert.Equal("ABCDEFGHIJKLMNOPQRST"u8.ToArray(), payload); + } + + [Fact] + public void BackReferenceBeyondTheWindowIsRejected() + { + // token 0x00 (lit 0); dist 0x0064 = 100 > the window (0 bytes before the first match in this block) + var lz = Synthetic.Concat([0x00], [0x00, 0x64], [0x04]); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Synthetic.Block(PlaBlockMode.Lz, lz, unpacked: 64)))); + + var ex = Assert.Throws(() => document.ReadPayload(document.Blocks.Single())); + Assert.Contains("back-reference distance 100", ex.Message); + } + + [Fact] + public void TruncatedTokenStreamIsRejected() + { + // token 0x40 declares 4 literals; the payload carries only 2 + var lz = Synthetic.Concat([0x40], [0x77, 0x78]); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Synthetic.Block(PlaBlockMode.Lz, lz, unpacked: 4)))); + + Assert.Throws(() => document.ReadPayload(document.Blocks.Single())); + } +} diff --git a/tests/PlaIO.Tests/PlaDocumentTests.cs b/tests/PlaIO.Tests/PlaDocumentTests.cs index df4f2fd..4dd7c25 100644 --- a/tests/PlaIO.Tests/PlaDocumentTests.cs +++ b/tests/PlaIO.Tests/PlaDocumentTests.cs @@ -13,14 +13,15 @@ private static byte[] OneBlockOfEachMode() => Synthetic.Fdb( Synthetic.Block(PlaBlockMode.Lz, new byte[12], tag: "DY", unpacked: 40)); [Fact] - public void ReadsModesZeroAndOneAndRefusesModeTwo() + public void ReadsAllThreePayloadModes() { using var document = PlaDocument.Open(new MemoryStream(OneBlockOfEachMode())); Assert.Equal([PlaBlockMode.Raw, PlaBlockMode.ZeroMask, PlaBlockMode.Lz], document.Blocks.Select(b => b.Mode)); Assert.Equal(RawPayload, document.ReadPayload(document.Blocks[0])); Assert.Equal(Plain, document.ReadPayload(document.Blocks[1])); - var e = Assert.Throws(() => document.ReadPayload(document.Blocks[2])); - Assert.Contains("mode 2", e.Message); + // The mode-2 payload (12 zero bytes) is a token stream: token 0x00 (no literals), + // distance 0 (emit zeros), length byte 0 (256) clipped at the declared size of 40. + Assert.Equal(new byte[40], document.ReadPayload(document.Blocks[2])); } [Fact] diff --git a/tests/PlaIO.Tests/SampleTests.cs b/tests/PlaIO.Tests/SampleTests.cs index 533b0ba..61d7473 100644 --- a/tests/PlaIO.Tests/SampleTests.cs +++ b/tests/PlaIO.Tests/SampleTests.cs @@ -137,14 +137,15 @@ public void NearlyEveryZeroMaskBlockDecodesToItsDeclaredSize() } [SkippableFact] - public void ReadingAnLzBlockThrowsNotSupported() + public void EveryLzBlockDecodesAndOnlyFalseEntriesAreSkipped() { - var (sample, block) = corpus.Fdb - .SelectMany(s => s.Document.Blocks.Where(b => b.Mode == PlaBlockMode.Lz).Select(b => (s, b))) - .FirstOrDefault(); - Assert.True(block != null, "No sample has a mode-2 block."); - var e = Assert.Throws(() => sample.Document.ReadPayload(block)); - Assert.Contains("mode 2", e.Message); + foreach (var sample in corpus.Fdb) + { + sample.Document.DecodeGlobalWindow(out var skipped); + string where = $"{sample.Name}: {skipped.Count} of {sample.Document.Blocks.Count} blocks skipped"; + Assert.True(skipped.All(b => b.Mode != PlaBlockMode.Lz), $"{where}; a mode-2 block did not decode"); + Assert.True(skipped.Count <= 0.01 * sample.Document.Blocks.Count, where); + } } [SkippableFact] From 0262b01003b0afff5c61f0d35bb80e1db51726a4 Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:57:37 +0300 Subject: [PATCH 3/4] Drop the obsolete synthetic chunk helpers --- tests/PlaIO.Tests/Synthetic.cs | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/tests/PlaIO.Tests/Synthetic.cs b/tests/PlaIO.Tests/Synthetic.cs index 94c1ae4..75979fd 100644 --- a/tests/PlaIO.Tests/Synthetic.cs +++ b/tests/PlaIO.Tests/Synthetic.cs @@ -125,30 +125,6 @@ public static byte[] RandomBytes(Random random, int length) /// A container: the 32-byte header, then the parts from offset 32. public static byte[] Fdb(params byte[][] parts) => Header().Concat(parts.SelectMany(p => p)).ToArray(); - /// A mode-2 chunk header followed by . Long header (9 bytes) when - /// ; bit 0 of the flags marks the body compressed. - public static byte[] LzChunk(bool compressed, bool longHeader, long bodySize, long unpacked, byte[]? body = null) - { - byte flags = (byte)((compressed ? 0x01 : 0x00) | (longHeader ? 0x02 : 0x00) | 0x44); - var bytes = new List { flags }; - if (longHeader) - { - bytes.AddRange(Le32((uint)bodySize + 9)); - bytes.AddRange(Le32((uint)unpacked)); - } - else - { - if (bodySize + 3 > 255 || unpacked > 255) throw new ArgumentOutOfRangeException(nameof(bodySize), "A short-header chunk holds at most 252 body bytes."); - bytes.Add((byte)(bodySize + 3)); - bytes.Add((byte)unpacked); - } - bytes.AddRange(body ?? new byte[bodySize]); - return bytes.ToArray(); - } - - /// An LZ payload: the chunk chain from the parts, then zero bytes. - public static byte[] LzPayload(byte[][] chunks, int tail = 0) => Concat(chunks).Concat(new byte[tail]).ToArray(); - /// A PNG: the signature and each chunk as [BE length][type][data][CRC]; the CRC is zero (not checked). public static byte[] Png(params (string Type, byte[] Data)[] chunks) { From e9878d13eaa947714d7d0d6570d956f9b9aa75a2 Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:53:23 +0300 Subject: [PATCH 4/4] Refine the mode 2 format: stream headers, length-less final matches, buffer blocks Three refinements established on the big sample corpus (85,429 mode 2 blocks of five files decode with zero failures; the six controlled saves still decode exactly): - A payload may start with a 9-byte stream header [u8][u32][u32]: present when the u32 at offset 5 equals the block's packed or unpacked size, the token stream starts after it. Without skipping it the parse "self-synchronizes" through the header and occasionally converges on the wrong reading - the old chunk-chain artifact came from exactly this. - A final match may omit the length byte: the body ends after the distance and the match length is the remainder of the output. - Some blocks declare an output buffer larger than the stream produces (sizes near whole mebibytes; the stream just ends with the body). The produced bytes fill the head of the buffer, the tail stays zero, and the global window holds the buffer space so directory positions keep addressing it. Detected by retrying the decode with the relaxed ending when the strict decode runs out of body. A stream that ends with the body is therefore a valid buffer block, not truncation; a back-reference beyond the decoded stream remains the one hard error. --- CHANGELOG.md | 15 +- docs/public-api.md | 2 +- src/PlaIO/Internal/LzPayloadDecoder.cs | 197 ++++++++++++++++--------- src/PlaIO/PlaDocument.cs | 18 ++- tests/PlaIO.Tests/LzPayloadTests.cs | 63 +++++++- 5 files changed, 213 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 045a45c..865a60d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,15 @@ API between minor releases. ### Added -- `PlaDocument.ReadPayload` decodes mode 2 (LZ) payloads. The mode 2 token stream produces exactly the block's declared - unpacked size and may reference the decompressed output of any preceding block of the file, so `ReadPayload` for a - mode 2 block decodes the file from the start into one global window; the cost of the call grows with the size of the - file. Preceding blocks that do not decode (false entries of the scan) are skipped and leave the window unchanged. A - stream that does not produce the declared size, or a back-reference beyond the decoded stream, throws - `InvalidDataException`. +- `PlaDocument.ReadPayload` decodes mode 2 (LZ) payloads. A payload may start with a 9-byte stream header holding the + packed and/or unpacked size. The token stream produces the block's declared unpacked size and may reference the + decompressed output of any preceding block of the file, so `ReadPayload` for a mode 2 block decodes the file from the + start into one global window; the cost of the call grows with the size of the file. Preceding blocks that do not + decode (false entries of the scan) are skipped and leave the window unchanged. A final match may omit the length byte + (the match length is the remainder of the output). Some blocks declare an output buffer larger than the stream + produces: the stream simply ends with the body, the produced bytes fill the head of the buffer and the tail stays + zero — the global window holds the buffer space, so directory positions keep addressing it. A back-reference beyond + the decoded stream throws `InvalidDataException`. ### Changed diff --git a/docs/public-api.md b/docs/public-api.md index 6b94506..0fb864f 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -21,7 +21,7 @@ The `PlaIO` package is the supported surface listed here; every public type is i | `PlaDocument.Length` | Length of the file in bytes. | | `PlaDocument.Header` | The `FdbHeader`. | | `PlaDocument.Blocks` | The block index in file order, found by scanning from offset 32. No on-disk block map is known, so completeness is not claimed. The scan can take a byte run for a block header, so the index may contain false entries. In real files a small share of mode 1 blocks does not decode to its declared size, and `ReadPayload` throws `InvalidDataException` for them. | -| `PlaDocument.ReadPayload(PlaBlock block)` | The decoded payload of a block of this document. Decodes all three storage modes: raw (mode 0), zero masks (mode 1) and the LZ token stream (mode 2). A mode 2 payload is produced by decoding every preceding block of the file in order into one global window (the token stream may reference the output of any preceding block), so the cost of the call grows with the size of the file; preceding blocks that do not decode (false entries of the scan) are skipped and leave the window unchanged. Throws `InvalidDataException` when a payload does not decode to the declared size or the file is cut short, and `ArgumentException` when the block is not from this document. | +| `PlaDocument.ReadPayload(PlaBlock block)` | The decoded payload of a block of this document. Decodes all three storage modes: raw (mode 0), zero masks (mode 1) and the LZ token stream (mode 2). A mode 2 payload may start with a 9-byte stream header (packed and/or unpacked size). The token stream produces the block's declared unpacked size — some blocks declare a buffer larger than the stream produces, the tail of the buffer stays zero — and may reference the output of any preceding block, so the payload is produced by decoding every preceding block of the file in order into one global window; the cost of the call grows with the size of the file. Preceding blocks that do not decode (false entries of the scan) are skipped and leave the window unchanged. Throws `InvalidDataException` when a back-reference reaches beyond the decoded stream or the file is cut short, and `ArgumentException` when the block is not from this document. | | `PlaDocument.FindDirectories()` | Searches every raw (mode 0) block large enough for 64 records of 30 bytes for an object directory, at most one per block. Recomputed on each call; mode 1 payloads are not searched. | | `PlaDocument.Dispose()` | Releases the stream unless the document was opened with `leaveOpen`. | | `FdbHeader` | `Size` (64), `LayoutVersion` (u16 at offset 8), `FormatVersion` (u16 at offset 10), `Field12` (u32 at offset 12; meaning not documented), `Raw` (the first min(64, file length) bytes; bytes 32..63 overlap the first block). | diff --git a/src/PlaIO/Internal/LzPayloadDecoder.cs b/src/PlaIO/Internal/LzPayloadDecoder.cs index 0021fff..92693ad 100644 --- a/src/PlaIO/Internal/LzPayloadDecoder.cs +++ b/src/PlaIO/Internal/LzPayloadDecoder.cs @@ -1,82 +1,147 @@ -using System.Buffers.Binary; - namespace PlaIO.Internal; -/// Decodes the mode 2 (LZ) token stream of one block payload into the -/// block output (exactly the block's declared unpacked size), appending every -/// produced byte to the window — the global decompressed stream of the whole -/// file. Back-references reach anywhere inside that stream, including the output of -/// preceding blocks. +/// Decodes the mode 2 (LZ) token stream of one block payload and appends every produced byte to the window — +/// the global decompressed stream of the whole file. Back-references reach anywhere inside that stream, including the +/// output of preceding blocks. Returns the number of body bytes consumed. /// -/// The stream is a chain of sequences. A sequence starts with a token byte: the high nibble -/// is the number of literal bytes that follow (15 = extended: the following bytes are added while -/// they equal 255). After the literals, if the output is not complete yet, come the match fields: -/// a 16-bit big-endian distance and a length byte. A distance of 0 emits the match length as zero -/// bytes (RLE); a positive distance copies the match length from that many bytes back in the -/// global window, byte by byte, so overlapping copies replicate a repeating pattern. The low -/// nibble of the token is the high byte of the match length, so a match spans up to 4095 bytes -/// (a length byte of 0 with a non-zero high nibble still means 256). When the output is full the -/// sequence ends; the remaining bytes of the body are slack the writer reserved and are not -/// consumed. +/// The payload may start with a 9-byte stream header [u8][u32 packed][u32 unpacked]: it is present when the +/// u32 at offset 5 equals the block's declared unpacked size, and the token stream starts after it. +/// The stream is a chain of sequences. A sequence starts with a token byte: the high nibble is the number of +/// literal bytes that follow (15 = extended: the following bytes are added while they equal 255). After the literals, if +/// the output is not complete yet, come the match fields: a 16-bit big-endian distance and a length byte. A distance of 0 +/// emits the match length as zero bytes (RLE); a positive distance copies the match length from that many bytes back in +/// the global window, byte by byte, so overlapping copies replicate a repeating pattern. The low nibble of the token is +/// the high byte of the match length, so a match spans up to 4095 bytes (a length byte of 0 with a non-zero high nibble +/// still means 256). When the output is full the sequence ends; the remaining bytes of the body are slack the writer +/// reserved and are not consumed. +/// Two endings depart from the strict scheme. A final match may omit the length byte: the body ends after the +/// distance, and the match length is the remainder of the output. And some blocks (repeating kinds, sizes near whole +/// mebibytes) declare an output buffer larger than the stream actually produces: the stream simply ends with the body, +/// and the real output is what was produced so far. Such a block is detected by retrying with this relaxed ending when +/// the strict decode runs out of body; the produced count then fills only part of the output and the caller pads the +/// rest (the directory addresses the buffer space, so the block's contribution to the global window is the declared +/// size). A back-reference that reaches beyond the decoded stream is always an error. /// internal static class LzPayloadDecoder { - public static int Decode(ReadOnlySpan body, Span output, List window, long blockOffset) + /// Decodes one mode 2 payload. Returns the consumed source length; receives the + /// number of output bytes the stream actually generated (equal to output.Length except for a buffer block). + public static int Decode(ReadOnlySpan body, Span output, List window, long blockOffset, out int produced) + { + var result = DecodeAttempt(body, output, window, blockOffset, bufferMode: false); + if (!result.ok) + { + result = DecodeAttempt(body, output, window, blockOffset, bufferMode: true); + if (!result.ok) + throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: {result.error}"); + } + produced = result.produced; + return result.consumed; + } + + private static (bool ok, int consumed, int produced, string error) DecodeAttempt( + ReadOnlySpan body, Span output, List window, long blockOffset, bool bufferMode) { int src = 0, o = 0; - while (o < output.Length) + int windowStart = window.Count; + uint declared = (uint)output.Length; + if (body.Length >= 9) { - if (src >= body.Length) - throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the token stream ends after {o} of {output.Length} output bytes."); - int token = body[src++]; - int litLen = token >> 4; - if (litLen == 15) + uint field = body[5] | ((uint)body[6] << 8) | ((uint)body[7] << 16) | ((uint)body[8] << 24); + if (field == declared || field == (uint)body.Length) + src = 9; // stream header [u8][u32][u32] holding the packed and/or unpacked size, the token stream starts after it + } + + void RollBack() + { + window.RemoveRange(windowStart, window.Count - windowStart); + } + + try + { + while (o < output.Length) { - byte x; - do + if (src >= body.Length) { - if (src >= body.Length) - throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the literal length extension runs past the payload."); - x = body[src++]; - litLen += x; + if (bufferMode) break; // the stream simply ends with the body + throw new InvalidDataException($"the token stream ends after {o} of {output.Length} output bytes."); + } + int token = body[src++]; + int litLen = token >> 4; + if (litLen == 15) + { + byte x; + do + { + if (src >= body.Length) + { + if (bufferMode) return End(o, src, output.Length); // the stream simply ends with the body + throw new InvalidDataException("the literal length extension runs past the payload."); + } + x = body[src++]; + litLen += x; + } + while (x == 255 && src < body.Length); + } + int litTake = Math.Min(litLen, body.Length - src); + int litEnd = Math.Min(o + litLen, output.Length); + litTake = Math.Min(litTake, litEnd - o); + for (int i = 0; i < litTake; i++, o++) + { + byte v = body[src++]; + output[o] = v; + window.Add(v); + } + if (o >= output.Length) + return End(o, src, output.Length); // trailing literals completed the block; the body tail is slack + if (litTake < litLen && src >= body.Length) + { + // the declared literal run reaches past the body + if (bufferMode) return End(o, src, output.Length); // the stream simply ends with the body + throw new InvalidDataException($"the literal run of {litLen} bytes reaches past the payload at {src} ({o} of {output.Length} output bytes)."); + } + if (src + 2 > body.Length) + { + if (bufferMode) return End(o, src, output.Length); // the stream simply ends with the body + throw new InvalidDataException($"the distance of the sequence at {src - 1} runs past the payload ({o} of {output.Length} output bytes)."); + } + int dist = (body[src] << 8) | body[src + 1]; + src += 2; + int matchLen; + if (src >= body.Length) + { + // final match without a length byte: the match length is the remainder of the output + matchLen = output.Length - o; + } + else + { + int lenByte = body[src++]; + matchLen = lenByte == 0 ? 256 : (((token & 0xF) << 8) | lenByte); + } + if (o + matchLen > output.Length) + matchLen = output.Length - o; // the final match is clipped at the declared size + if (dist == 0) + { + for (int i = 0; i < matchLen; i++, o++) { output[o] = 0; window.Add(0); } + continue; + } + if (dist > window.Count) + throw new InvalidDataException($"the back-reference distance {dist} exceeds the decoded stream ({window.Count} bytes)."); + for (int i = 0; i < matchLen; i++, o++) + { + byte v = window[window.Count - dist]; + output[o] = v; + window.Add(v); } - while (x == 255 && src < body.Length); - } - if (src + litLen > body.Length) - throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the literal run of {litLen} bytes runs past the payload at {src}."); - int litEnd = Math.Min(o + litLen, output.Length); - for (; o < litEnd; o++) - { - byte v = body[src++]; - output[o] = v; - window.Add(v); - } - if (o >= output.Length) - return src; // trailing literals completed the block; the body tail is slack - if (src + 2 > body.Length) - throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the distance of the sequence at {src - 1} runs past the payload."); - int dist = (body[src] << 8) | body[src + 1]; - src += 2; - if (src >= body.Length) - throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the match length of the sequence at {src - 2} runs past the payload."); - int lenByte = body[src++]; - int matchLen = lenByte == 0 ? 256 : (((token & 0xF) << 8) | lenByte); - if (o + matchLen > output.Length) - matchLen = output.Length - o; // the final match is clipped at the declared size - if (dist == 0) - { - for (int i = 0; i < matchLen; i++, o++) { output[o] = 0; window.Add(0); } - continue; - } - if (dist > window.Count) - throw new InvalidDataException($"Mode-2 block at 0x{blockOffset:x}: the back-reference distance {dist} exceeds the decoded stream ({window.Count} bytes)."); - for (int i = 0; i < matchLen; i++, o++) - { - byte v = window[window.Count - dist]; - output[o] = v; - window.Add(v); } + return End(o, src, output.Length); + } + catch (InvalidDataException e) + { + RollBack(); + return (false, src, o, e.Message); } - return src; + + static (bool, int, int, string) End(int o, int src, int declared) => (true, src, o, null!); } } diff --git a/src/PlaIO/PlaDocument.cs b/src/PlaIO/PlaDocument.cs index 52997d1..9f4b87b 100644 --- a/src/PlaIO/PlaDocument.cs +++ b/src/PlaIO/PlaDocument.cs @@ -137,17 +137,20 @@ internal byte[] ReadPayload(PlaBlock block, out int consumed) } var blockPayload = new byte[block.PackedSize]; StreamReads.ReadExactlyAt(_stream, block.PayloadOffset, blockPayload, $"the payload of the block at 0x{block.Offset:x}"); - // Mode 2 (LZ): the token stream produces exactly the declared unpacked size and may reference the - // window; the body tail after the output is complete is writer slack. + // Mode 2 (LZ): the token stream produces the declared unpacked size (some blocks declare a buffer + // larger than the stream produces — the tail of the buffer stays zero) and may reference the window; + // the body tail after the output is complete is writer slack. var output = new byte[block.UnpackedSize]; + int lzConsumed; try { - consumed = LzPayloadDecoder.Decode(blockPayload, output, window, block.Offset); + lzConsumed = LzPayloadDecoder.Decode(blockPayload, output, window, block.Offset, out _); } catch (InvalidDataException e) { throw new InvalidDataException($"Block at 0x{block.Offset:x}: {e.Message}", e); } + consumed = lzConsumed; return output; } default: @@ -214,10 +217,13 @@ private void AppendToWindow(PlaBlock b, List window) window.AddRange(decoded); return; } - // Mode 2 (LZ): the token stream produces exactly the declared unpacked size and the decoder appends - // every produced byte to the window; the body tail after the output is complete is writer slack. + // Mode 2 (LZ): the token stream produces the declared unpacked size (some blocks declare a buffer + // larger than the stream produces — the tail of the buffer stays zero) and the decoder appends every + // produced byte to the window; the body tail after the output is complete is writer slack. var output = new byte[b.UnpackedSize]; - LzPayloadDecoder.Decode(payload, output, window, b.Offset); + int produced; + LzPayloadDecoder.Decode(payload, output, window, b.Offset, out produced); + for (long i = produced; i < b.UnpackedSize; i++) window.Add(0); // the directory addresses the buffer space } /// Searches every raw block large enough for 64 records for an object directory (at most one per block). diff --git a/tests/PlaIO.Tests/LzPayloadTests.cs b/tests/PlaIO.Tests/LzPayloadTests.cs index f0ee5ab..b1dfd07 100644 --- a/tests/PlaIO.Tests/LzPayloadTests.cs +++ b/tests/PlaIO.Tests/LzPayloadTests.cs @@ -98,13 +98,70 @@ public void BackReferenceBeyondTheWindowIsRejected() } [Fact] - public void TruncatedTokenStreamIsRejected() + public void AStreamEndingWithTheBodyIsABufferBlockEvenWithoutAStreamHeader() { - // token 0x40 declares 4 literals; the payload carries only 2 + // token 0x40 declares 4 literals; the payload carries only 2 and ends: the stream ends with the + // body, so the block decodes as a buffer block and the payload holds the declared buffer. var lz = Synthetic.Concat([0x40], [0x77, 0x78]); using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( Synthetic.Block(PlaBlockMode.Lz, lz, unpacked: 4)))); - Assert.Throws(() => document.ReadPayload(document.Blocks.Single())); + var payload = document.ReadPayload(document.Blocks.Single()); + Assert.Equal(new byte[] { 0x77, 0x78, 0, 0 }, payload); + } + + [Fact] + public void StreamHeaderIsSkippedWhenItsSizeFieldMatches() + { + // [u8][u32 packed][u32 unpacked][token stream]: the u32 at offset 5 equals the declared size, + // so the token stream starts after the 9-byte header. + var header = Synthetic.Concat([0x47], BitConverter.GetBytes(15u), BitConverter.GetBytes(4u)); + var lz = Synthetic.Concat(header, [0x40], "ABCD"u8.ToArray(), [0xFF, 0xFE]); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Synthetic.Block(PlaBlockMode.Lz, lz, unpacked: 4)))); + + Assert.Equal("ABCD"u8.ToArray(), document.ReadPayload(document.Blocks.Single())); + } + + [Fact] + public void FinalMatchWithoutLengthByteClipsToTheDeclaredSize() + { + // token 0x40 (lit 4) "ABCD"; dist 4 and the body ends: the match length is the remainder + // of the output, so the copy fills exactly the declared 8 bytes. + var lz = Synthetic.Concat([0x40], "ABCD"u8.ToArray(), [0x00, 0x04]); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Synthetic.Block(PlaBlockMode.Lz, lz, unpacked: 8)))); + + Assert.Equal("ABCDABCD"u8.ToArray(), document.ReadPayload(document.Blocks.Single())); + } + + [Fact] + public void ABufferBlockProducesItsActualStreamAndZeroTail() + { + // the token stream produces 4 bytes, but the block declares a 16-byte buffer: the tail of the + // buffer stays zero and the payload returns the whole buffer. + var lz = Synthetic.Concat([0x40], "ABCD"u8.ToArray()); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Synthetic.Block(PlaBlockMode.Lz, lz, unpacked: 16)))); + + var payload = document.ReadPayload(document.Blocks.Single()); + Assert.Equal(16, payload.Length); + Assert.Equal("ABCD"u8.ToArray(), payload[..4]); + Assert.Equal(new byte[12], payload[4..]); + } + + [Fact] + public void TheWindowHoldsTheBufferSpaceOfAPrecedingBufferBlock() + { + // block A is a buffer block (actual "ABCD", declared 16): the window receives the buffer space + // (ABCD + 12 zeros). Block B references the window tail: dist 6 lands in the zero padding. + var a = Synthetic.Concat([0x40], "ABCD"u8.ToArray()); + var b = Synthetic.Concat([0x40], "wxyz"u8.ToArray(), [0x00, 0x06], [0x04]); + using var document = PlaDocument.Open(new MemoryStream(Synthetic.Fdb( + Synthetic.Block(PlaBlockMode.Lz, a, unpacked: 16), + Synthetic.Block(PlaBlockMode.Lz, b, unpacked: 8)))); + + var payload = document.ReadPayload(document.Blocks[1]); + Assert.Equal(new byte[] { (byte)'w', (byte)'x', (byte)'y', (byte)'z', 0, 0, (byte)'w', (byte)'x' }, payload); } }