From 278449f9e5bf21e4d6bae98fa2cd20da9df89b75 Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Mon, 16 Nov 2026 02:10:38 +0000 Subject: [PATCH 1/2] Report the incomplete results of a Leica container --- CHANGELOG.md | 3 + README.md | 10 ++- docs/public-api.md | 2 +- src/PclIO/Formats/Leica/LgsxDocument.cs | 106 ++++++++++++++++++++++-- src/PclIO/PclReadOptions.cs | 7 ++ tests/PclIO.Tests/FormatTests.cs | 65 +++++++++++++++ tests/PclIO.Tests/SyntheticFixtures.cs | 24 +++++- 7 files changed, 199 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c472fd6..9cf62eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `ApplyTransform` is false. A record that does not hold usable values is left unread, and the setup keeps an unknown pose. - The point count of a Leica scan is taken from the container when it states one, so that reading it does not decode the point data. Only a container without that number still needs the decoding. +- The Leica reader counts the data it cannot use (the points that are dropped as padding, the compressed streams that are + skipped and the point data that ends at an unknown block) and reports it in the warnings of the document. The new + `PclReadOptions.Strict` makes such a read fail with `InvalidDataException` instead. ## [0.1.0] - 2026-11-15 diff --git a/README.md b/README.md index 2221728..c5ad003 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,9 @@ These readers drop data without an error or a warning, so a scan can yield fewer data file in neither layout, or in the sectioned layout with an element type or header value the reader does not support, cannot be read: the project lists it with a warning and reading its points throws `NotSupportedException`. A compressed stream whose copies reach back more than 1 MiB is not decoded either: reading its points throws `NotSupportedException`, without an earlier warning. -- Leica: padding entries and points that coincide with a block's origin can be dropped, a compressed stream above its size limit - is skipped, and a block that is not recognised ends the scan. The result can therefore be incomplete. +- Leica: padding entries and points that coincide with a block's origin are dropped, a compressed stream above its size limit is + skipped, and a block that is not recognised ends the scan. The reader counts what it dropped or skipped and reports it in the + warnings of the document, so that an incomplete result is visible. With `PclReadOptions.Strict` the read fails instead. - EdgeWise: a database node whose point count is 0 is skipped. ### Limits @@ -98,8 +99,9 @@ These readers drop data without an error or a warning, so a scan can yield fewer A file that exceeds one of these limits, or whose declared sizes and counts do not fit the file, is rejected with `InvalidDataException`, with the exceptions below. A FARO scan grid above its limit gives `NotSupportedException` when the file is opened. An E57 point prototype with more than 1024 fields gives `NotSupportedException` when the points are read; the scan still -lists. A compressed Leica stream above its size limit is not rejected: its data is skipped, which is one of the Leica data filters -above. An unsupported PinPoint feature (see the data filters) gives `NotSupportedException` when the points are read. Point data is streamed and there is no limit on the file size, except for FARO container files: each compressed data stream is +lists. A compressed Leica stream above its size limit is not rejected: its data is skipped and the skip is reported in the warnings +of the document, which is one of the Leica data filters above. An unsupported PinPoint feature (see the data filters) gives +`NotSupportedException` when the points are read. Point data is streamed and there is no limit on the file size, except for FARO container files: each compressed data stream is read into memory as a whole, so its size must fit a .NET array (about 2 GB), otherwise reading fails with `NotSupportedException`. The XML of E57 and ReCap project files is read with DTDs prohibited, so no entity is expanded and nothing outside the file is fetched. diff --git a/docs/public-api.md b/docs/public-api.md index c2a8928..57bbedc 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -15,7 +15,7 @@ format-specific members can change when the support for that format changes. |---|---| | `PclFile` | Static entry point. `DetectKind(path)` and `DetectKind(stream, extension)` return a `PclFileKind`; `Open(path, options)` detects the format and returns a `PclDocument`. | | `PclFileKind` | `Unknown`, `E57`, `RecapProject`, `RecapScan`, `LeicaLgsx`, `LeicaLgs`, `FaroFls`, `PinPoint` and `EdgeWise`. | -| `PclReadOptions` | `MaxPoints` (limit of one read call, null for all, 0 for none), `SubsampleStep` (1 keeps every point, N keeps about one in N) and `ApplyTransform` (default true). Negative `MaxPoints` and a step below 1 throw `ArgumentOutOfRangeException`. `PclReadOptions.Default` has no limit. | +| `PclReadOptions` | `MaxPoints` (limit of one read call, null for all, 0 for none), `SubsampleStep` (1 keeps every point, N keeps about one in N), `ApplyTransform` (default true) and `Strict` (default false: a reader that finds incomplete data fails with `InvalidDataException` instead of adding a warning). Negative `MaxPoints` and a step below 1 throw `ArgumentOutOfRangeException`. `PclReadOptions.Default` has no limit. | `Open` throws `FileNotFoundException` for a missing path, `DirectoryNotFoundException` when a directory the reader needs is missing or disappears while reading, `NotSupportedException` for an unrecognised format or an unsupported feature, and diff --git a/src/PclIO/Formats/Leica/LgsxDocument.cs b/src/PclIO/Formats/Leica/LgsxDocument.cs index 1158f58..c4d54b0 100644 --- a/src/PclIO/Formats/Leica/LgsxDocument.cs +++ b/src/PclIO/Formats/Leica/LgsxDocument.cs @@ -70,6 +70,15 @@ private LgsxDocument(string path, long fileSize, string guid, string version, bo _metadata["Format"] = isLegacy ? "Leica LGS" : "Leica LGSx"; } + /// + /// Adds a warning to the document. The same message is added only once, so that reading the same scan twice + /// does not repeat it. + /// + internal void AddWarning(string message) + { + if (!_warnings.Contains(message, StringComparer.Ordinal)) _warnings.Add(message); + } + /// Opens a Leica container and reads its table of contents. /// Path of the .lgsx or .lgs file. /// Read options; currently unused when opening. @@ -235,7 +244,8 @@ public static LgsxDocument Open(string path, PclReadOptions? options = null) blockStarts: blockStarts, transform: setup.Translation is { } translation ? new PclTransform { Translation = translation } : null, bounds: setup.Bounds, - declaredPointCount: setup.PointCount >= 0 ? setup.PointCount : null)); + declaredPointCount: setup.PointCount >= 0 ? setup.PointCount : null, + document: document)); } document._warnings.Add( @@ -251,7 +261,8 @@ public static LgsxDocument Open(string path, PclReadOptions? options = null) name: Path.GetFileNameWithoutExtension(path), path: path, pointCloudEntry: pointCloudEntry, - declaredPointCount: declaredCount)); + declaredPointCount: declaredCount, + document: document)); } if (scanSetups.Count > 0) @@ -522,6 +533,7 @@ public sealed class LgsxScan : PclScan private readonly string _filePath; private readonly LgsxEntry? _entry; + private readonly LgsxDocument? _document; // Start offsets of the blocks that belong to this scan; null when the scan holds all blocks of the container. private readonly long[]? _blockStarts; @@ -532,6 +544,12 @@ public sealed class LgsxScan : PclScan private long _pointCount = -1; private (bool Colors, bool Intensities, bool Normals)? _channels; + // What the last read did not return; the counters belong to one read and are reported when it is done. + private long _droppedPoints; + private long _skippedStreams; + private long _blocksRead; + private bool _chainEndedEarly; + /// public override string Id { get; } @@ -558,7 +576,7 @@ public sealed class LgsxScan : PclScan private (bool Colors, bool Intensities, bool Normals) Channels => _channels ??= DetectChannels(); internal LgsxScan(string id, string name, string path, LgsxEntry? pointCloudEntry, long[]? blockStarts = null, - PclTransform? transform = null, PclBounds? bounds = null, long? declaredPointCount = null) + PclTransform? transform = null, PclBounds? bounds = null, long? declaredPointCount = null, LgsxDocument? document = null) { Id = id; Name = name; @@ -566,6 +584,7 @@ internal LgsxScan(string id, string name, string path, LgsxEntry? pointCloudEntr _entry = pointCloudEntry; _blockStarts = blockStarts; _declaredPointCount = declaredPointCount; + _document = document; if (transform != null) Transform = transform; if (bounds != null) Bounds = bounds.Value; } @@ -591,13 +610,21 @@ private sealed record Block(double ScaleX, double ScaleY, double ScaleZ, double /// Streams the points of this scan. /// + /// /// Positions are decoded from the packed position planes of each block. A code of zero at any slot except the /// first of a block is treated as unused padding and is not returned, so a real point that sits exactly at the /// block origin is dropped. A block yields as many points as its shortest position plane holds. Returned points /// keep consecutive values. + /// + /// + /// Data that the reader cannot use is counted and reported in the warnings of the document: the points that were + /// dropped as padding, the compressed streams that were skipped because they are too large and the point data + /// that ends at a block the reader does not know. With the read fails with + /// instead of the warning. + /// /// /// Read options; null uses defaults. - /// The point data is malformed or declares oversized data. + /// The point data is malformed, declares oversized data or is incomplete in strict mode. public override IEnumerable ReadPoints(PclReadOptions? options = null) { options ??= PclReadOptions.Default; @@ -608,6 +635,12 @@ public override IEnumerable ReadPoints(PclReadOptions? options = null) int step = Math.Max(1, options.SubsampleStep); long yielded = 0; + // The counters describe this read; they start at zero every time. + _droppedPoints = 0; + _skippedStreams = 0; + _blocksRead = 0; + _chainEndedEarly = false; + using var fs = File.OpenRead(_filePath); foreach (var block in ReadBlocks(fs)) { @@ -629,7 +662,12 @@ public override IEnumerable ReadPoints(PclReadOptions? options = null) if (i % step != 0) continue; uint val = (uint)(p0[i] | (p1[i] << 8) | (p2[i] << 16) | (p3[i] << 24)); - if (val == 0 && i > 0 && p0[i] == 0 && p1[i] == 0) continue; + if (val == 0 && i > 0 && p0[i] == 0 && p1[i] == 0) + { + // The slot holds no point: it is padding or a point at the origin of the block. + _droppedPoints++; + continue; + } // Unpack the three 10-bit coordinates from the packed code. uint xm = 0, ym = 0, zm = 0; @@ -694,6 +732,35 @@ public override IEnumerable ReadPoints(PclReadOptions? options = null) if (yielded >= targetCount) yield break; } } + + // The whole point data was read, so what it did not hold can be reported. + ReportIncompleteData(options.Strict); + } + + /// + /// Reports the data that the point data did not hold: the points that were dropped as padding, the compressed + /// streams that were skipped and the point data that ends at a block the reader does not know. In strict mode + /// the read fails instead. + /// + private void ReportIncompleteData(bool strict) + { + var problems = new List(); + + if (_droppedPoints > 0) + problems.Add($"{_droppedPoints} points were dropped because their slot holds no point"); + + if (_skippedStreams > 0) + problems.Add($"{_skippedStreams} compressed streams were skipped because they are larger than the accepted size"); + + if (_chainEndedEarly) + problems.Add($"the point data ends after {_blocksRead} blocks at a block that is not a point block"); + + if (problems.Count == 0) return; + + string message = "The point data is incomplete: " + string.Join("; ", problems) + "."; + if (strict) throw FormatError.Corrupted(message); + + _document?.AddWarning(message); } /// Walks the chain of point blocks and yields those that carry point planes and belong to this scan. @@ -704,7 +771,10 @@ private IEnumerable ReadBlocks(FileStream fs) long decodedBytes = 0; byte[] blockHeader = new byte[128]; - foreach (var (blockStart, blockLength) in LgsxDocument.ReadBlockChain(fs, _entry!)) + var chain = LgsxDocument.ReadBlockChain(fs, _entry!); + long payloadEnd = (long)_entry!.Offset + (long)_entry.Size; + + foreach (var (blockStart, blockLength) in chain) { if (!BelongsToScan(blockStart)) continue; @@ -753,17 +823,26 @@ private IEnumerable ReadBlocks(FileStream fs) double.IsFinite(minX) && double.IsFinite(minY) && double.IsFinite(minZ))) throw FormatError.Corrupted("point block has an invalid coordinate scale or origin."); + _blocksRead++; yield return new Block(scaleX, scaleY, scaleZ, minX, minY, minZ, frames); } } } + + // Data that is left in the point data but that is not a point block means that the reader did not read + // everything the container holds. + if (chain.Count > 0) + { + long lastEnd = chain[^1].Start + chain[^1].Length; + if (payloadEnd - lastEnd >= 8 + 128) _chainEndedEarly = true; + } } // A scan without its own blocks holds all of them; otherwise only the blocks the container assigned to it. private bool BelongsToScan(long blockStart) => _blockStarts == null || Array.BinarySearch(_blockStarts, blockStart) >= 0; - private static List ReadAllFrames(FileStream fs, long offset, long maxOffset) + private List ReadAllFrames(FileStream fs, long offset, long maxOffset) { var list = new List(); long limit = Math.Min(maxOffset, fs.Length); @@ -777,9 +856,18 @@ private static List ReadAllFrames(FileStream fs, long offset, long maxOf { fs.ReadExactly(lenBuf); - // A zero or out-of-range length marks the end of the frame list. + // A zero length marks the end of the frame list. ulong fLen = Bytes.ReadUInt64LE(lenBuf); - if (fLen == 0 || fLen > MaxCompressedFrameBytes || (long)fLen > limit - fs.Position) break; + if (fLen == 0) break; + + // A stream that is larger than the accepted size is skipped: the rest of the block is lost with it. + if (fLen > MaxCompressedFrameBytes) + { + _skippedStreams++; + break; + } + + if ((long)fLen > limit - fs.Position) break; byte[] fData = new byte[(int)fLen]; fs.ReadExactly(fData); diff --git a/src/PclIO/PclReadOptions.cs b/src/PclIO/PclReadOptions.cs index 5b2a964..a10620b 100644 --- a/src/PclIO/PclReadOptions.cs +++ b/src/PclIO/PclReadOptions.cs @@ -45,4 +45,11 @@ public int SubsampleStep /// or the 4x4 matrix when the format supplies one. Rotation values are not applied. /// public bool ApplyTransform { get; init; } = true; + + /// + /// Whether a reader that finds incomplete data fails instead of reporting it: a reader that would add a warning + /// about dropped or skipped data throws then. Readers that cannot detect + /// incomplete data ignore this option. The default is false, so that a read never fails because of it. + /// + public bool Strict { get; init; } } diff --git a/tests/PclIO.Tests/FormatTests.cs b/tests/PclIO.Tests/FormatTests.cs index 809a983..62b5a13 100644 --- a/tests/PclIO.Tests/FormatTests.cs +++ b/tests/PclIO.Tests/FormatTests.cs @@ -503,6 +503,71 @@ public void The_Point_Count_Of_A_Lgsx_Scan_Does_Not_Decode_The_Points() Assert.Throws(() => scan.ReadPoints().ToList()); } + [Fact] + public void A_Lgsx_Container_Reports_The_Points_That_Were_Dropped() + { + // The second point has the code of the origin of the block, so it is dropped as padding. + var points = new List + { + new(1, 2, 3), + new(0, 0, 0), + new(4, 5, 6), + new(0, 0, 0), + }; + + using var dir = new TempDirectory(); + using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(points))); + + var scan = Assert.Single(doc.Scans); + Assert.Equal(2, scan.ReadPoints().Count()); + Assert.Contains(doc.Warnings, w => w.Contains("2 points were dropped", StringComparison.Ordinal)); + } + + [Fact] + public void A_Lgsx_Container_Reports_The_Compressed_Streams_That_Were_Skipped() + { + using var dir = new TempDirectory(); + using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(oversizedFrame: true))); + + var scan = Assert.Single(doc.Scans); + + // The frames before the oversized one are read, the rest of the block is lost with it. + Assert.Equal(5, scan.ReadPoints().Count()); + Assert.Contains(doc.Warnings, w => w.Contains("1 compressed streams were skipped", StringComparison.Ordinal)); + } + + [Fact] + public void A_Lgsx_Container_Reports_Point_Data_That_Ends_At_A_Block_It_Does_Not_Know() + { + using var dir = new TempDirectory(); + using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(unknownBlock: true))); + + var scan = Assert.Single(doc.Scans); + Assert.Equal(5, scan.ReadPoints().Count()); + Assert.Contains(doc.Warnings, w => w.Contains("ends after 1 blocks", StringComparison.Ordinal)); + } + + [Fact] + public void A_Lgsx_Container_Fails_In_Strict_Mode_When_Its_Point_Data_Is_Incomplete() + { + using var dir = new TempDirectory(); + using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(unknownBlock: true))); + + var scan = Assert.Single(doc.Scans); + Assert.Throws(() => scan.ReadPoints(new PclReadOptions { Strict = true }).ToList()); + } + + [Fact] + public void A_Lgsx_Container_With_Complete_Point_Data_Reports_Nothing() + { + using var dir = new TempDirectory(); + using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx())); + + var scan = Assert.Single(doc.Scans); + Assert.Equal(5, scan.ReadPoints().Count()); + Assert.DoesNotContain(doc.Warnings, w => w.Contains("incomplete", StringComparison.Ordinal)); + } + [Fact] public void A_Lgsx_Block_Is_Not_Cut_At_65536_Points() { diff --git a/tests/PclIO.Tests/SyntheticFixtures.cs b/tests/PclIO.Tests/SyntheticFixtures.cs index 6a09096..9b5ced0 100644 --- a/tests/PclIO.Tests/SyntheticFixtures.cs +++ b/tests/PclIO.Tests/SyntheticFixtures.cs @@ -155,9 +155,12 @@ public static byte[] CreateZip(string entryName, string text) /// True writes a setup record whose translation is not a number. /// True writes point blocks whose compressed frames cannot be decoded. /// False leaves the number of points out of the header of the point data. + /// True writes a frame whose declared length is above the accepted size. + /// True writes a block after the point blocks that the reader does not know. public static byte[] CreateSyntheticLgsx(IReadOnlyList? points = null, bool withColorAndIntensity = false, bool withPointData = true, IReadOnlyList? extraPlanes = null, int setupCount = 1, bool assignPointsToSetups = false, bool overlappingSetupRanges = false, - bool collidingSetupNames = false, bool damagedSetupRecord = false, bool damagedPointData = false, bool withPointCount = true) + bool collidingSetupNames = false, bool damagedSetupRecord = false, bool damagedPointData = false, bool withPointCount = true, + bool oversizedFrame = false, bool unknownBlock = false) { points ??= DefaultLeicaPoints; using var ms = new MemoryStream(); @@ -171,7 +174,7 @@ public static byte[] CreateSyntheticLgsx(IReadOnlyList? points : new List> { points }; (byte[] payload, List<(long Start, long Length)> blocks) = - BuildLeicaPayload(groups, withColorAndIntensity, extraPlanes ?? [], damagedPointData, withPointCount); + BuildLeicaPayload(groups, withColorAndIntensity, extraPlanes ?? [], damagedPointData, withPointCount, oversizedFrame, unknownBlock); // The point data follows the container entries and their records, so its start is known before they are written. var setupNames = Enumerable.Range(0, setupCount).Select(i => "SETUP-" + (3 + i)).ToList(); @@ -299,7 +302,7 @@ private static void WriteEntry(BinaryWriter bw, string name, ulong payloadOffset // Builds the point data payload: a header, then one block per group of points. private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLeicaPayload( IReadOnlyList> groups, bool full, IReadOnlyList extraPlanes, bool damaged = false, - bool withPointCount = true) + bool withPointCount = true, bool oversizedFrame = false, bool unknownBlock = false) { using var ms = new MemoryStream(); using var bw = new BinaryWriter(ms); @@ -324,7 +327,10 @@ private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLei // A frame whose descriptor is not valid: the header of the block stays readable, the data does not. foreach (byte[] broken in frames) broken[4] = 0x5A; } - int blockLength = 8 + 128 + frames.Sum(f => 8 + f.Length); + + // A frame whose declared length is above the accepted size is skipped by the reader. + const ulong OversizedLength = 20_000_000; + int blockLength = 8 + 128 + frames.Sum(f => 8 + f.Length) + (oversizedFrame ? 8 : 0); long start = ms.Position; bw.Write("APRN"u8.ToArray()); @@ -340,9 +346,19 @@ private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLei bw.Write(f); } + if (oversizedFrame) bw.Write(OversizedLength); + blocks.Add((start, blockLength)); } + // A chunk that is not a point block: the reader stops the chain there. + if (unknownBlock) + { + bw.Write("XPRB"u8.ToArray()); + bw.Write((uint)(8 + 128)); + bw.Write(new byte[128]); + } + return (ms.ToArray(), blocks); } From 1fdf7327a3eeaa6700b886339ccaf8e3be5a1e9c Mon Sep 17 00:00:00 2001 From: mahach666 <79989386+mahach666@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:02:58 +0000 Subject: [PATCH 2/2] Report every incomplete path of the Leica point data and keep the counters per read --- src/PclIO/Formats/Leica/LgsxDocument.cs | 118 ++++++++++++++++-------- tests/PclIO.Tests/FormatTests.cs | 51 ++++++++++ tests/PclIO.Tests/SyntheticFixtures.cs | 25 ++++- 3 files changed, 152 insertions(+), 42 deletions(-) diff --git a/src/PclIO/Formats/Leica/LgsxDocument.cs b/src/PclIO/Formats/Leica/LgsxDocument.cs index c4d54b0..9e8aaaf 100644 --- a/src/PclIO/Formats/Leica/LgsxDocument.cs +++ b/src/PclIO/Formats/Leica/LgsxDocument.cs @@ -72,11 +72,14 @@ private LgsxDocument(string path, long fileSize, string guid, string version, bo /// /// Adds a warning to the document. The same message is added only once, so that reading the same scan twice - /// does not repeat it. + /// does not repeat it. Reads of different scans may add warnings at the same time, so the list is guarded. /// internal void AddWarning(string message) { - if (!_warnings.Contains(message, StringComparer.Ordinal)) _warnings.Add(message); + lock (_warnings) + { + if (!_warnings.Contains(message, StringComparer.Ordinal)) _warnings.Add(message); + } } /// Opens a Leica container and reads its table of contents. @@ -544,11 +547,20 @@ public sealed class LgsxScan : PclScan private long _pointCount = -1; private (bool Colors, bool Intensities, bool Normals)? _channels; - // What the last read did not return; the counters belong to one read and are reported when it is done. - private long _droppedPoints; - private long _skippedStreams; - private long _blocksRead; - private bool _chainEndedEarly; + /// + /// What one read did not return. Each enumeration keeps its own state, so interleaved or parallel reads of the + /// same scan do not mix their counts; the state is reported when the read is done. + /// + private sealed class ReadState + { + public long DroppedPoints; + public long SkippedStreams; + public long BlocksRead; + public long UnusableBlocks; + public long IncompleteBlocks; + public bool ChainEndedEarly; + public long ChainBlocks; + } /// public override string Id { get; } @@ -599,7 +611,7 @@ private sealed record Block(double ScaleX, double ScaleY, double ScaleZ, double if (_entry == null) return (false, false, false); using var fs = File.OpenRead(_filePath); - foreach (var block in ReadBlocks(fs)) + foreach (var block in ReadBlocks(fs, new ReadState())) { int n = block.Frames.Count; return (n >= ColorFrames, n >= IntensityFrames, n >= NormalFrames); @@ -618,8 +630,9 @@ private sealed record Block(double ScaleX, double ScaleY, double ScaleZ, double /// /// /// Data that the reader cannot use is counted and reported in the warnings of the document: the points that were - /// dropped as padding, the compressed streams that were skipped because they are too large and the point data - /// that ends at a block the reader does not know. With the read fails with + /// dropped as padding, the compressed streams that were skipped because they are too large, the point blocks that + /// carry no positions or whose frame list ends before the block does, and the point data that ends at a block the + /// reader does not know. With the read fails with /// instead of the warning. /// /// @@ -635,14 +648,11 @@ public override IEnumerable ReadPoints(PclReadOptions? options = null) int step = Math.Max(1, options.SubsampleStep); long yielded = 0; - // The counters describe this read; they start at zero every time. - _droppedPoints = 0; - _skippedStreams = 0; - _blocksRead = 0; - _chainEndedEarly = false; + // The counters describe this read alone; they start at zero every time. + var state = new ReadState(); using var fs = File.OpenRead(_filePath); - foreach (var block in ReadBlocks(fs)) + foreach (var block in ReadBlocks(fs, state)) { var frames = block.Frames; byte[] p0 = frames[0]; @@ -665,7 +675,7 @@ public override IEnumerable ReadPoints(PclReadOptions? options = null) if (val == 0 && i > 0 && p0[i] == 0 && p1[i] == 0) { // The slot holds no point: it is padding or a point at the origin of the block. - _droppedPoints++; + state.DroppedPoints++; continue; } @@ -734,26 +744,32 @@ public override IEnumerable ReadPoints(PclReadOptions? options = null) } // The whole point data was read, so what it did not hold can be reported. - ReportIncompleteData(options.Strict); + ReportIncompleteData(options.Strict, state); } /// /// Reports the data that the point data did not hold: the points that were dropped as padding, the compressed - /// streams that were skipped and the point data that ends at a block the reader does not know. In strict mode - /// the read fails instead. + /// streams that were skipped, the point blocks that carry no positions or whose frame list is cut short, and the + /// point data that ends at a block the reader does not know. In strict mode the read fails instead. /// - private void ReportIncompleteData(bool strict) + private void ReportIncompleteData(bool strict, ReadState state) { var problems = new List(); - if (_droppedPoints > 0) - problems.Add($"{_droppedPoints} points were dropped because their slot holds no point"); + if (state.DroppedPoints > 0) + problems.Add($"{state.DroppedPoints} points were dropped because their slot holds no point"); + + if (state.SkippedStreams > 0) + problems.Add($"{state.SkippedStreams} compressed streams were skipped because they are larger than the accepted size"); + + if (state.UnusableBlocks > 0) + problems.Add($"{state.UnusableBlocks} point blocks were skipped because they hold fewer than four planes"); - if (_skippedStreams > 0) - problems.Add($"{_skippedStreams} compressed streams were skipped because they are larger than the accepted size"); + if (state.IncompleteBlocks > 0) + problems.Add($"{state.IncompleteBlocks} point blocks have a frame list that ends before the block does"); - if (_chainEndedEarly) - problems.Add($"the point data ends after {_blocksRead} blocks at a block that is not a point block"); + if (state.ChainEndedEarly) + problems.Add($"the point data ends after {state.ChainBlocks} blocks at a block that is not a point block"); if (problems.Count == 0) return; @@ -764,7 +780,7 @@ private void ReportIncompleteData(bool strict) } /// Walks the chain of point blocks and yields those that carry point planes and belong to this scan. - private IEnumerable ReadBlocks(FileStream fs) + private IEnumerable ReadBlocks(FileStream fs, ReadState state) { long fileLength = fs.Length; long decodedLimit = DecodedAllowanceBytes + (fileLength * DecodedBytesPerFileByte); @@ -808,7 +824,7 @@ private IEnumerable ReadBlocks(FileStream fs) if (firstFrame >= 8) { long firstFrameOffset = blockStart + firstFrame; - var frames = ReadAllFrames(fs, firstFrameOffset, blockEnd); + var frames = ReadAllFrames(fs, firstFrameOffset, blockEnd, state); // Many small blocks that each expand a lot would otherwise make the file cost unbounded time. foreach (byte[] plane in frames) decodedBytes += plane.Length; @@ -823,18 +839,35 @@ private IEnumerable ReadBlocks(FileStream fs) double.IsFinite(minX) && double.IsFinite(minY) && double.IsFinite(minZ))) throw FormatError.Corrupted("point block has an invalid coordinate scale or origin."); - _blocksRead++; + state.BlocksRead++; yield return new Block(scaleX, scaleY, scaleZ, minX, minY, minZ, frames); } + else state.UnusableBlocks++; } + else state.UnusableBlocks++; } // Data that is left in the point data but that is not a point block means that the reader did not read - // everything the container holds. - if (chain.Count > 0) + // everything the container holds. The gap belongs to the payload, so only the unified scan or the scan + // that owns the last block reports it, and the count is the count of the whole chain. + state.ChainBlocks = chain.Count; + if (_blockStarts == null || (chain.Count > 0 && BelongsToScan(chain[^1].Start))) { - long lastEnd = chain[^1].Start + chain[^1].Length; - if (payloadEnd - lastEnd >= 8 + 128) _chainEndedEarly = true; + if (chain.Count > 0) + { + long lastEnd = chain[^1].Start + chain[^1].Length; + if (payloadEnd - lastEnd >= 8 + 128) state.ChainEndedEarly = true; + } + else + { + // An empty chain means the very first chunk is not a point block. When a full block could have + // fit there, the point data ends at a block the reader does not know, just as with a later one. + fs.Position = (long)_entry!.Offset; + byte[] payloadHeader = new byte[64]; + fs.ReadExactly(payloadHeader); + long firstBlock = (long)_entry.Offset + Bytes.ReadUInt32LE(payloadHeader.AsSpan(0x20, 4)); + if (payloadEnd - firstBlock >= 8 + 128) state.ChainEndedEarly = true; + } } } @@ -842,7 +875,7 @@ private IEnumerable ReadBlocks(FileStream fs) private bool BelongsToScan(long blockStart) => _blockStarts == null || Array.BinarySearch(_blockStarts, blockStart) >= 0; - private List ReadAllFrames(FileStream fs, long offset, long maxOffset) + private List ReadAllFrames(FileStream fs, long offset, long maxOffset, ReadState state) { var list = new List(); long limit = Math.Min(maxOffset, fs.Length); @@ -863,16 +896,25 @@ private List ReadAllFrames(FileStream fs, long offset, long maxOffset) // A stream that is larger than the accepted size is skipped: the rest of the block is lost with it. if (fLen > MaxCompressedFrameBytes) { - _skippedStreams++; + state.SkippedStreams++; break; } - if ((long)fLen > limit - fs.Position) break; + // A frame list that ends before the block does leaves the rest of the block unread. + if ((long)fLen > limit - fs.Position) + { + state.IncompleteBlocks++; + break; + } byte[] fData = new byte[(int)fLen]; fs.ReadExactly(fData); - if (!fData.AsSpan().StartsWith(FrameMagic)) break; + if (!fData.AsSpan().StartsWith(FrameMagic)) + { + state.IncompleteBlocks++; + break; + } if (list.Count >= MaxFramesPerBlock) throw FormatError.Corrupted($"point block has more than {MaxFramesPerBlock} planes."); diff --git a/tests/PclIO.Tests/FormatTests.cs b/tests/PclIO.Tests/FormatTests.cs index 62b5a13..bb42511 100644 --- a/tests/PclIO.Tests/FormatTests.cs +++ b/tests/PclIO.Tests/FormatTests.cs @@ -547,6 +547,57 @@ public void A_Lgsx_Container_Reports_Point_Data_That_Ends_At_A_Block_It_Does_Not Assert.Contains(doc.Warnings, w => w.Contains("ends after 1 blocks", StringComparison.Ordinal)); } + [Fact] + public void A_Lgsx_Container_Whose_First_Chunk_Is_Not_A_Point_Block_Reports_It() + { + // The chain is empty because the very first chunk is unknown; the point data still ends at a block the + // reader does not know, and the declared count is not what a read returns. + using var dir = new TempDirectory(); + using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(unknownFirstBlock: true))); + + var scan = Assert.Single(doc.Scans); + Assert.Equal(5, scan.PointCount); + Assert.Empty(scan.ReadPoints()); + Assert.Contains(doc.Warnings, w => w.Contains("ends after 0 blocks", StringComparison.Ordinal)); + Assert.Throws(() => scan.ReadPoints(new PclReadOptions { Strict = true }).ToList()); + } + + [Fact] + public void A_Lgsx_Block_Whose_Frame_List_Is_Cut_Short_Is_Reported() + { + using var dir = new TempDirectory(); + using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(truncatedFrameList: true))); + + var scan = Assert.Single(doc.Scans); + Assert.Equal(5, scan.ReadPoints().Count()); + Assert.Contains(doc.Warnings, w => w.Contains("1 point blocks have a frame list that ends before the block does", StringComparison.Ordinal)); + } + + [Fact] + public void A_Lgsx_Block_With_Fewer_Than_Four_Planes_Is_Reported() + { + using var dir = new TempDirectory(); + using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(fewPlanes: true))); + + var scan = Assert.Single(doc.Scans); + Assert.Empty(scan.ReadPoints()); + Assert.Contains(doc.Warnings, w => w.Contains("1 point blocks were skipped because they hold fewer than four planes", StringComparison.Ordinal)); + } + + [Fact] + public void Interleaved_Reads_Of_A_Lgsx_Scan_Keep_Their_Own_Counts() + { + // Two enumerators advance side by side; the counters of one read must not reset or mix with the other's. + var points = new List { new(1, 2, 3), new(0, 0, 0), new(4, 5, 6), new(0, 0, 0) }; + using var dir = new TempDirectory(); + using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(points))); + + var scan = Assert.Single(doc.Scans); + var pairs = scan.ReadPoints().Zip(scan.ReadPoints(), (a, b) => (a, b)).ToList(); + Assert.Equal(2, pairs.Count); + Assert.Contains(doc.Warnings, w => w.Contains("2 points were dropped", StringComparison.Ordinal)); + } + [Fact] public void A_Lgsx_Container_Fails_In_Strict_Mode_When_Its_Point_Data_Is_Incomplete() { diff --git a/tests/PclIO.Tests/SyntheticFixtures.cs b/tests/PclIO.Tests/SyntheticFixtures.cs index 9b5ced0..2509a4e 100644 --- a/tests/PclIO.Tests/SyntheticFixtures.cs +++ b/tests/PclIO.Tests/SyntheticFixtures.cs @@ -157,10 +157,13 @@ public static byte[] CreateZip(string entryName, string text) /// False leaves the number of points out of the header of the point data. /// True writes a frame whose declared length is above the accepted size. /// True writes a block after the point blocks that the reader does not know. + /// True writes a chunk the reader does not know before the point blocks. + /// True ends the frame list of a block with a length prefix that declares more bytes than the block holds. + /// True writes the blocks with fewer than four planes, so they carry no positions. public static byte[] CreateSyntheticLgsx(IReadOnlyList? points = null, bool withColorAndIntensity = false, bool withPointData = true, IReadOnlyList? extraPlanes = null, int setupCount = 1, bool assignPointsToSetups = false, bool overlappingSetupRanges = false, bool collidingSetupNames = false, bool damagedSetupRecord = false, bool damagedPointData = false, bool withPointCount = true, - bool oversizedFrame = false, bool unknownBlock = false) + bool oversizedFrame = false, bool unknownBlock = false, bool unknownFirstBlock = false, bool truncatedFrameList = false, bool fewPlanes = false) { points ??= DefaultLeicaPoints; using var ms = new MemoryStream(); @@ -174,7 +177,8 @@ public static byte[] CreateSyntheticLgsx(IReadOnlyList? points : new List> { points }; (byte[] payload, List<(long Start, long Length)> blocks) = - BuildLeicaPayload(groups, withColorAndIntensity, extraPlanes ?? [], damagedPointData, withPointCount, oversizedFrame, unknownBlock); + BuildLeicaPayload(groups, withColorAndIntensity, extraPlanes ?? [], damagedPointData, withPointCount, oversizedFrame, unknownBlock, + unknownFirstBlock, truncatedFrameList, fewPlanes); // The point data follows the container entries and their records, so its start is known before they are written. var setupNames = Enumerable.Range(0, setupCount).Select(i => "SETUP-" + (3 + i)).ToList(); @@ -302,7 +306,8 @@ private static void WriteEntry(BinaryWriter bw, string name, ulong payloadOffset // Builds the point data payload: a header, then one block per group of points. private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLeicaPayload( IReadOnlyList> groups, bool full, IReadOnlyList extraPlanes, bool damaged = false, - bool withPointCount = true, bool oversizedFrame = false, bool unknownBlock = false) + bool withPointCount = true, bool oversizedFrame = false, bool unknownBlock = false, + bool unknownFirstBlock = false, bool truncatedFrameList = false, bool fewPlanes = false) { using var ms = new MemoryStream(); using var bw = new BinaryWriter(ms); @@ -315,6 +320,14 @@ private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLei BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(0x28, 8), (ulong)groups.Sum(g => g.Count)); bw.Write(header); + // A chunk that is not a point block before the point blocks: the chain stays empty. + if (unknownFirstBlock) + { + bw.Write("XPRB"u8.ToArray()); + bw.Write((uint)(8 + 128)); + bw.Write(new byte[128]); + } + var blocks = new List<(long Start, long Length)>(); foreach (var group in groups) { @@ -322,6 +335,7 @@ private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLei if (group.Count == 0) continue; var frames = BuildFrames(group, full, extraPlanes); + if (fewPlanes) frames = [.. frames.Take(3)]; if (damaged) { // A frame whose descriptor is not valid: the header of the block stays readable, the data does not. @@ -330,7 +344,7 @@ private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLei // A frame whose declared length is above the accepted size is skipped by the reader. const ulong OversizedLength = 20_000_000; - int blockLength = 8 + 128 + frames.Sum(f => 8 + f.Length) + (oversizedFrame ? 8 : 0); + int blockLength = 8 + 128 + frames.Sum(f => 8 + f.Length) + (oversizedFrame ? 8 : 0) + (truncatedFrameList ? 8 : 0); long start = ms.Position; bw.Write("APRN"u8.ToArray()); @@ -348,6 +362,9 @@ private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLei if (oversizedFrame) bw.Write(OversizedLength); + // A length prefix that declares more bytes than the block holds: the frame list is cut short. + if (truncatedFrameList) bw.Write((ulong)1000); + blocks.Add((start, blockLength)); }