From 278449f9e5bf21e4d6bae98fa2cd20da9df89b75 Mon Sep 17 00:00:00 2001
From: mahach666 <79989386+mahach666@users.noreply.github.com>
Date: Mon, 16 Nov 2026 02:10:38 +0000
Subject: [PATCH 1/2] Report the incomplete results of a Leica container
---
CHANGELOG.md | 3 +
README.md | 10 ++-
docs/public-api.md | 2 +-
src/PclIO/Formats/Leica/LgsxDocument.cs | 106 ++++++++++++++++++++++--
src/PclIO/PclReadOptions.cs | 7 ++
tests/PclIO.Tests/FormatTests.cs | 65 +++++++++++++++
tests/PclIO.Tests/SyntheticFixtures.cs | 24 +++++-
7 files changed, 199 insertions(+), 18 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c472fd6..9cf62eb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`ApplyTransform` is false. A record that does not hold usable values is left unread, and the setup keeps an unknown pose.
- The point count of a Leica scan is taken from the container when it states one, so that reading it does not decode the
point data. Only a container without that number still needs the decoding.
+- The Leica reader counts the data it cannot use (the points that are dropped as padding, the compressed streams that are
+ skipped and the point data that ends at an unknown block) and reports it in the warnings of the document. The new
+ `PclReadOptions.Strict` makes such a read fail with `InvalidDataException` instead.
## [0.1.0] - 2026-11-15
diff --git a/README.md b/README.md
index 2221728..c5ad003 100644
--- a/README.md
+++ b/README.md
@@ -89,8 +89,9 @@ These readers drop data without an error or a warning, so a scan can yield fewer
data file in neither layout, or in the sectioned layout with an element type or header value the reader does not support, cannot
be read: the project lists it with a warning and reading its points throws `NotSupportedException`. A compressed stream whose
copies reach back more than 1 MiB is not decoded either: reading its points throws `NotSupportedException`, without an earlier warning.
-- Leica: padding entries and points that coincide with a block's origin can be dropped, a compressed stream above its size limit
- is skipped, and a block that is not recognised ends the scan. The result can therefore be incomplete.
+- Leica: padding entries and points that coincide with a block's origin are dropped, a compressed stream above its size limit is
+ skipped, and a block that is not recognised ends the scan. The reader counts what it dropped or skipped and reports it in the
+ warnings of the document, so that an incomplete result is visible. With `PclReadOptions.Strict` the read fails instead.
- EdgeWise: a database node whose point count is 0 is skipped.
### Limits
@@ -98,8 +99,9 @@ These readers drop data without an error or a warning, so a scan can yield fewer
A file that exceeds one of these limits, or whose declared sizes and counts do not fit the file, is rejected with
`InvalidDataException`, with the exceptions below. A FARO scan grid above its limit gives `NotSupportedException` when the file is
opened. An E57 point prototype with more than 1024 fields gives `NotSupportedException` when the points are read; the scan still
-lists. A compressed Leica stream above its size limit is not rejected: its data is skipped, which is one of the Leica data filters
-above. An unsupported PinPoint feature (see the data filters) gives `NotSupportedException` when the points are read. Point data is streamed and there is no limit on the file size, except for FARO container files: each compressed data stream is
+lists. A compressed Leica stream above its size limit is not rejected: its data is skipped and the skip is reported in the warnings
+of the document, which is one of the Leica data filters above. An unsupported PinPoint feature (see the data filters) gives
+`NotSupportedException` when the points are read. Point data is streamed and there is no limit on the file size, except for FARO container files: each compressed data stream is
read into memory as a whole, so its size must fit a .NET array (about 2 GB), otherwise reading fails with
`NotSupportedException`. The XML of E57 and ReCap project files is read with DTDs prohibited, so no entity is expanded and nothing
outside the file is fetched.
diff --git a/docs/public-api.md b/docs/public-api.md
index c2a8928..57bbedc 100644
--- a/docs/public-api.md
+++ b/docs/public-api.md
@@ -15,7 +15,7 @@ format-specific members can change when the support for that format changes.
|---|---|
| `PclFile` | Static entry point. `DetectKind(path)` and `DetectKind(stream, extension)` return a `PclFileKind`; `Open(path, options)` detects the format and returns a `PclDocument`. |
| `PclFileKind` | `Unknown`, `E57`, `RecapProject`, `RecapScan`, `LeicaLgsx`, `LeicaLgs`, `FaroFls`, `PinPoint` and `EdgeWise`. |
-| `PclReadOptions` | `MaxPoints` (limit of one read call, null for all, 0 for none), `SubsampleStep` (1 keeps every point, N keeps about one in N) and `ApplyTransform` (default true). Negative `MaxPoints` and a step below 1 throw `ArgumentOutOfRangeException`. `PclReadOptions.Default` has no limit. |
+| `PclReadOptions` | `MaxPoints` (limit of one read call, null for all, 0 for none), `SubsampleStep` (1 keeps every point, N keeps about one in N), `ApplyTransform` (default true) and `Strict` (default false: a reader that finds incomplete data fails with `InvalidDataException` instead of adding a warning). Negative `MaxPoints` and a step below 1 throw `ArgumentOutOfRangeException`. `PclReadOptions.Default` has no limit. |
`Open` throws `FileNotFoundException` for a missing path, `DirectoryNotFoundException` when a directory the reader needs is
missing or disappears while reading, `NotSupportedException` for an unrecognised format or an unsupported feature, and
diff --git a/src/PclIO/Formats/Leica/LgsxDocument.cs b/src/PclIO/Formats/Leica/LgsxDocument.cs
index 1158f58..c4d54b0 100644
--- a/src/PclIO/Formats/Leica/LgsxDocument.cs
+++ b/src/PclIO/Formats/Leica/LgsxDocument.cs
@@ -70,6 +70,15 @@ private LgsxDocument(string path, long fileSize, string guid, string version, bo
_metadata["Format"] = isLegacy ? "Leica LGS" : "Leica LGSx";
}
+ ///
+ /// Adds a warning to the document. The same message is added only once, so that reading the same scan twice
+ /// does not repeat it.
+ ///
+ internal void AddWarning(string message)
+ {
+ if (!_warnings.Contains(message, StringComparer.Ordinal)) _warnings.Add(message);
+ }
+
/// Opens a Leica container and reads its table of contents.
/// Path of the .lgsx or .lgs file.
/// Read options; currently unused when opening.
@@ -235,7 +244,8 @@ public static LgsxDocument Open(string path, PclReadOptions? options = null)
blockStarts: blockStarts,
transform: setup.Translation is { } translation ? new PclTransform { Translation = translation } : null,
bounds: setup.Bounds,
- declaredPointCount: setup.PointCount >= 0 ? setup.PointCount : null));
+ declaredPointCount: setup.PointCount >= 0 ? setup.PointCount : null,
+ document: document));
}
document._warnings.Add(
@@ -251,7 +261,8 @@ public static LgsxDocument Open(string path, PclReadOptions? options = null)
name: Path.GetFileNameWithoutExtension(path),
path: path,
pointCloudEntry: pointCloudEntry,
- declaredPointCount: declaredCount));
+ declaredPointCount: declaredCount,
+ document: document));
}
if (scanSetups.Count > 0)
@@ -522,6 +533,7 @@ public sealed class LgsxScan : PclScan
private readonly string _filePath;
private readonly LgsxEntry? _entry;
+ private readonly LgsxDocument? _document;
// Start offsets of the blocks that belong to this scan; null when the scan holds all blocks of the container.
private readonly long[]? _blockStarts;
@@ -532,6 +544,12 @@ public sealed class LgsxScan : PclScan
private long _pointCount = -1;
private (bool Colors, bool Intensities, bool Normals)? _channels;
+ // What the last read did not return; the counters belong to one read and are reported when it is done.
+ private long _droppedPoints;
+ private long _skippedStreams;
+ private long _blocksRead;
+ private bool _chainEndedEarly;
+
///
public override string Id { get; }
@@ -558,7 +576,7 @@ public sealed class LgsxScan : PclScan
private (bool Colors, bool Intensities, bool Normals) Channels => _channels ??= DetectChannels();
internal LgsxScan(string id, string name, string path, LgsxEntry? pointCloudEntry, long[]? blockStarts = null,
- PclTransform? transform = null, PclBounds? bounds = null, long? declaredPointCount = null)
+ PclTransform? transform = null, PclBounds? bounds = null, long? declaredPointCount = null, LgsxDocument? document = null)
{
Id = id;
Name = name;
@@ -566,6 +584,7 @@ internal LgsxScan(string id, string name, string path, LgsxEntry? pointCloudEntr
_entry = pointCloudEntry;
_blockStarts = blockStarts;
_declaredPointCount = declaredPointCount;
+ _document = document;
if (transform != null) Transform = transform;
if (bounds != null) Bounds = bounds.Value;
}
@@ -591,13 +610,21 @@ private sealed record Block(double ScaleX, double ScaleY, double ScaleZ, double
/// Streams the points of this scan.
///
+ ///
/// Positions are decoded from the packed position planes of each block. A code of zero at any slot except the
/// first of a block is treated as unused padding and is not returned, so a real point that sits exactly at the
/// block origin is dropped. A block yields as many points as its shortest position plane holds. Returned points
/// keep consecutive values.
+ ///
+ ///
+ /// Data that the reader cannot use is counted and reported in the warnings of the document: the points that were
+ /// dropped as padding, the compressed streams that were skipped because they are too large and the point data
+ /// that ends at a block the reader does not know. With the read fails with
+ /// instead of the warning.
+ ///
///
/// Read options; null uses defaults.
- /// The point data is malformed or declares oversized data.
+ /// The point data is malformed, declares oversized data or is incomplete in strict mode.
public override IEnumerable ReadPoints(PclReadOptions? options = null)
{
options ??= PclReadOptions.Default;
@@ -608,6 +635,12 @@ public override IEnumerable ReadPoints(PclReadOptions? options = null)
int step = Math.Max(1, options.SubsampleStep);
long yielded = 0;
+ // The counters describe this read; they start at zero every time.
+ _droppedPoints = 0;
+ _skippedStreams = 0;
+ _blocksRead = 0;
+ _chainEndedEarly = false;
+
using var fs = File.OpenRead(_filePath);
foreach (var block in ReadBlocks(fs))
{
@@ -629,7 +662,12 @@ public override IEnumerable ReadPoints(PclReadOptions? options = null)
if (i % step != 0) continue;
uint val = (uint)(p0[i] | (p1[i] << 8) | (p2[i] << 16) | (p3[i] << 24));
- if (val == 0 && i > 0 && p0[i] == 0 && p1[i] == 0) continue;
+ if (val == 0 && i > 0 && p0[i] == 0 && p1[i] == 0)
+ {
+ // The slot holds no point: it is padding or a point at the origin of the block.
+ _droppedPoints++;
+ continue;
+ }
// Unpack the three 10-bit coordinates from the packed code.
uint xm = 0, ym = 0, zm = 0;
@@ -694,6 +732,35 @@ public override IEnumerable ReadPoints(PclReadOptions? options = null)
if (yielded >= targetCount) yield break;
}
}
+
+ // The whole point data was read, so what it did not hold can be reported.
+ ReportIncompleteData(options.Strict);
+ }
+
+ ///
+ /// Reports the data that the point data did not hold: the points that were dropped as padding, the compressed
+ /// streams that were skipped and the point data that ends at a block the reader does not know. In strict mode
+ /// the read fails instead.
+ ///
+ private void ReportIncompleteData(bool strict)
+ {
+ var problems = new List();
+
+ if (_droppedPoints > 0)
+ problems.Add($"{_droppedPoints} points were dropped because their slot holds no point");
+
+ if (_skippedStreams > 0)
+ problems.Add($"{_skippedStreams} compressed streams were skipped because they are larger than the accepted size");
+
+ if (_chainEndedEarly)
+ problems.Add($"the point data ends after {_blocksRead} blocks at a block that is not a point block");
+
+ if (problems.Count == 0) return;
+
+ string message = "The point data is incomplete: " + string.Join("; ", problems) + ".";
+ if (strict) throw FormatError.Corrupted(message);
+
+ _document?.AddWarning(message);
}
/// Walks the chain of point blocks and yields those that carry point planes and belong to this scan.
@@ -704,7 +771,10 @@ private IEnumerable ReadBlocks(FileStream fs)
long decodedBytes = 0;
byte[] blockHeader = new byte[128];
- foreach (var (blockStart, blockLength) in LgsxDocument.ReadBlockChain(fs, _entry!))
+ var chain = LgsxDocument.ReadBlockChain(fs, _entry!);
+ long payloadEnd = (long)_entry!.Offset + (long)_entry.Size;
+
+ foreach (var (blockStart, blockLength) in chain)
{
if (!BelongsToScan(blockStart)) continue;
@@ -753,17 +823,26 @@ private IEnumerable ReadBlocks(FileStream fs)
double.IsFinite(minX) && double.IsFinite(minY) && double.IsFinite(minZ)))
throw FormatError.Corrupted("point block has an invalid coordinate scale or origin.");
+ _blocksRead++;
yield return new Block(scaleX, scaleY, scaleZ, minX, minY, minZ, frames);
}
}
}
+
+ // Data that is left in the point data but that is not a point block means that the reader did not read
+ // everything the container holds.
+ if (chain.Count > 0)
+ {
+ long lastEnd = chain[^1].Start + chain[^1].Length;
+ if (payloadEnd - lastEnd >= 8 + 128) _chainEndedEarly = true;
+ }
}
// A scan without its own blocks holds all of them; otherwise only the blocks the container assigned to it.
private bool BelongsToScan(long blockStart) =>
_blockStarts == null || Array.BinarySearch(_blockStarts, blockStart) >= 0;
- private static List ReadAllFrames(FileStream fs, long offset, long maxOffset)
+ private List ReadAllFrames(FileStream fs, long offset, long maxOffset)
{
var list = new List();
long limit = Math.Min(maxOffset, fs.Length);
@@ -777,9 +856,18 @@ private static List ReadAllFrames(FileStream fs, long offset, long maxOf
{
fs.ReadExactly(lenBuf);
- // A zero or out-of-range length marks the end of the frame list.
+ // A zero length marks the end of the frame list.
ulong fLen = Bytes.ReadUInt64LE(lenBuf);
- if (fLen == 0 || fLen > MaxCompressedFrameBytes || (long)fLen > limit - fs.Position) break;
+ if (fLen == 0) break;
+
+ // A stream that is larger than the accepted size is skipped: the rest of the block is lost with it.
+ if (fLen > MaxCompressedFrameBytes)
+ {
+ _skippedStreams++;
+ break;
+ }
+
+ if ((long)fLen > limit - fs.Position) break;
byte[] fData = new byte[(int)fLen];
fs.ReadExactly(fData);
diff --git a/src/PclIO/PclReadOptions.cs b/src/PclIO/PclReadOptions.cs
index 5b2a964..a10620b 100644
--- a/src/PclIO/PclReadOptions.cs
+++ b/src/PclIO/PclReadOptions.cs
@@ -45,4 +45,11 @@ public int SubsampleStep
/// or the 4x4 matrix when the format supplies one. Rotation values are not applied.
///
public bool ApplyTransform { get; init; } = true;
+
+ ///
+ /// Whether a reader that finds incomplete data fails instead of reporting it: a reader that would add a warning
+ /// about dropped or skipped data throws then. Readers that cannot detect
+ /// incomplete data ignore this option. The default is false, so that a read never fails because of it.
+ ///
+ public bool Strict { get; init; }
}
diff --git a/tests/PclIO.Tests/FormatTests.cs b/tests/PclIO.Tests/FormatTests.cs
index 809a983..62b5a13 100644
--- a/tests/PclIO.Tests/FormatTests.cs
+++ b/tests/PclIO.Tests/FormatTests.cs
@@ -503,6 +503,71 @@ public void The_Point_Count_Of_A_Lgsx_Scan_Does_Not_Decode_The_Points()
Assert.Throws(() => scan.ReadPoints().ToList());
}
+ [Fact]
+ public void A_Lgsx_Container_Reports_The_Points_That_Were_Dropped()
+ {
+ // The second point has the code of the origin of the block, so it is dropped as padding.
+ var points = new List
+ {
+ new(1, 2, 3),
+ new(0, 0, 0),
+ new(4, 5, 6),
+ new(0, 0, 0),
+ };
+
+ using var dir = new TempDirectory();
+ using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(points)));
+
+ var scan = Assert.Single(doc.Scans);
+ Assert.Equal(2, scan.ReadPoints().Count());
+ Assert.Contains(doc.Warnings, w => w.Contains("2 points were dropped", StringComparison.Ordinal));
+ }
+
+ [Fact]
+ public void A_Lgsx_Container_Reports_The_Compressed_Streams_That_Were_Skipped()
+ {
+ using var dir = new TempDirectory();
+ using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(oversizedFrame: true)));
+
+ var scan = Assert.Single(doc.Scans);
+
+ // The frames before the oversized one are read, the rest of the block is lost with it.
+ Assert.Equal(5, scan.ReadPoints().Count());
+ Assert.Contains(doc.Warnings, w => w.Contains("1 compressed streams were skipped", StringComparison.Ordinal));
+ }
+
+ [Fact]
+ public void A_Lgsx_Container_Reports_Point_Data_That_Ends_At_A_Block_It_Does_Not_Know()
+ {
+ using var dir = new TempDirectory();
+ using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(unknownBlock: true)));
+
+ var scan = Assert.Single(doc.Scans);
+ Assert.Equal(5, scan.ReadPoints().Count());
+ Assert.Contains(doc.Warnings, w => w.Contains("ends after 1 blocks", StringComparison.Ordinal));
+ }
+
+ [Fact]
+ public void A_Lgsx_Container_Fails_In_Strict_Mode_When_Its_Point_Data_Is_Incomplete()
+ {
+ using var dir = new TempDirectory();
+ using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(unknownBlock: true)));
+
+ var scan = Assert.Single(doc.Scans);
+ Assert.Throws(() => scan.ReadPoints(new PclReadOptions { Strict = true }).ToList());
+ }
+
+ [Fact]
+ public void A_Lgsx_Container_With_Complete_Point_Data_Reports_Nothing()
+ {
+ using var dir = new TempDirectory();
+ using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx()));
+
+ var scan = Assert.Single(doc.Scans);
+ Assert.Equal(5, scan.ReadPoints().Count());
+ Assert.DoesNotContain(doc.Warnings, w => w.Contains("incomplete", StringComparison.Ordinal));
+ }
+
[Fact]
public void A_Lgsx_Block_Is_Not_Cut_At_65536_Points()
{
diff --git a/tests/PclIO.Tests/SyntheticFixtures.cs b/tests/PclIO.Tests/SyntheticFixtures.cs
index 6a09096..9b5ced0 100644
--- a/tests/PclIO.Tests/SyntheticFixtures.cs
+++ b/tests/PclIO.Tests/SyntheticFixtures.cs
@@ -155,9 +155,12 @@ public static byte[] CreateZip(string entryName, string text)
/// True writes a setup record whose translation is not a number.
/// True writes point blocks whose compressed frames cannot be decoded.
/// False leaves the number of points out of the header of the point data.
+ /// True writes a frame whose declared length is above the accepted size.
+ /// True writes a block after the point blocks that the reader does not know.
public static byte[] CreateSyntheticLgsx(IReadOnlyList? points = null, bool withColorAndIntensity = false, bool withPointData = true,
IReadOnlyList? extraPlanes = null, int setupCount = 1, bool assignPointsToSetups = false, bool overlappingSetupRanges = false,
- bool collidingSetupNames = false, bool damagedSetupRecord = false, bool damagedPointData = false, bool withPointCount = true)
+ bool collidingSetupNames = false, bool damagedSetupRecord = false, bool damagedPointData = false, bool withPointCount = true,
+ bool oversizedFrame = false, bool unknownBlock = false)
{
points ??= DefaultLeicaPoints;
using var ms = new MemoryStream();
@@ -171,7 +174,7 @@ public static byte[] CreateSyntheticLgsx(IReadOnlyList? points
: new List> { points };
(byte[] payload, List<(long Start, long Length)> blocks) =
- BuildLeicaPayload(groups, withColorAndIntensity, extraPlanes ?? [], damagedPointData, withPointCount);
+ BuildLeicaPayload(groups, withColorAndIntensity, extraPlanes ?? [], damagedPointData, withPointCount, oversizedFrame, unknownBlock);
// The point data follows the container entries and their records, so its start is known before they are written.
var setupNames = Enumerable.Range(0, setupCount).Select(i => "SETUP-" + (3 + i)).ToList();
@@ -299,7 +302,7 @@ private static void WriteEntry(BinaryWriter bw, string name, ulong payloadOffset
// Builds the point data payload: a header, then one block per group of points.
private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLeicaPayload(
IReadOnlyList> groups, bool full, IReadOnlyList extraPlanes, bool damaged = false,
- bool withPointCount = true)
+ bool withPointCount = true, bool oversizedFrame = false, bool unknownBlock = false)
{
using var ms = new MemoryStream();
using var bw = new BinaryWriter(ms);
@@ -324,7 +327,10 @@ private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLei
// A frame whose descriptor is not valid: the header of the block stays readable, the data does not.
foreach (byte[] broken in frames) broken[4] = 0x5A;
}
- int blockLength = 8 + 128 + frames.Sum(f => 8 + f.Length);
+
+ // A frame whose declared length is above the accepted size is skipped by the reader.
+ const ulong OversizedLength = 20_000_000;
+ int blockLength = 8 + 128 + frames.Sum(f => 8 + f.Length) + (oversizedFrame ? 8 : 0);
long start = ms.Position;
bw.Write("APRN"u8.ToArray());
@@ -340,9 +346,19 @@ private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLei
bw.Write(f);
}
+ if (oversizedFrame) bw.Write(OversizedLength);
+
blocks.Add((start, blockLength));
}
+ // A chunk that is not a point block: the reader stops the chain there.
+ if (unknownBlock)
+ {
+ bw.Write("XPRB"u8.ToArray());
+ bw.Write((uint)(8 + 128));
+ bw.Write(new byte[128]);
+ }
+
return (ms.ToArray(), blocks);
}
From 1fdf7327a3eeaa6700b886339ccaf8e3be5a1e9c Mon Sep 17 00:00:00 2001
From: mahach666 <79989386+mahach666@users.noreply.github.com>
Date: Tue, 17 Nov 2026 13:02:58 +0000
Subject: [PATCH 2/2] Report every incomplete path of the Leica point data and
keep the counters per read
---
src/PclIO/Formats/Leica/LgsxDocument.cs | 118 ++++++++++++++++--------
tests/PclIO.Tests/FormatTests.cs | 51 ++++++++++
tests/PclIO.Tests/SyntheticFixtures.cs | 25 ++++-
3 files changed, 152 insertions(+), 42 deletions(-)
diff --git a/src/PclIO/Formats/Leica/LgsxDocument.cs b/src/PclIO/Formats/Leica/LgsxDocument.cs
index c4d54b0..9e8aaaf 100644
--- a/src/PclIO/Formats/Leica/LgsxDocument.cs
+++ b/src/PclIO/Formats/Leica/LgsxDocument.cs
@@ -72,11 +72,14 @@ private LgsxDocument(string path, long fileSize, string guid, string version, bo
///
/// Adds a warning to the document. The same message is added only once, so that reading the same scan twice
- /// does not repeat it.
+ /// does not repeat it. Reads of different scans may add warnings at the same time, so the list is guarded.
///
internal void AddWarning(string message)
{
- if (!_warnings.Contains(message, StringComparer.Ordinal)) _warnings.Add(message);
+ lock (_warnings)
+ {
+ if (!_warnings.Contains(message, StringComparer.Ordinal)) _warnings.Add(message);
+ }
}
/// Opens a Leica container and reads its table of contents.
@@ -544,11 +547,20 @@ public sealed class LgsxScan : PclScan
private long _pointCount = -1;
private (bool Colors, bool Intensities, bool Normals)? _channels;
- // What the last read did not return; the counters belong to one read and are reported when it is done.
- private long _droppedPoints;
- private long _skippedStreams;
- private long _blocksRead;
- private bool _chainEndedEarly;
+ ///
+ /// What one read did not return. Each enumeration keeps its own state, so interleaved or parallel reads of the
+ /// same scan do not mix their counts; the state is reported when the read is done.
+ ///
+ private sealed class ReadState
+ {
+ public long DroppedPoints;
+ public long SkippedStreams;
+ public long BlocksRead;
+ public long UnusableBlocks;
+ public long IncompleteBlocks;
+ public bool ChainEndedEarly;
+ public long ChainBlocks;
+ }
///
public override string Id { get; }
@@ -599,7 +611,7 @@ private sealed record Block(double ScaleX, double ScaleY, double ScaleZ, double
if (_entry == null) return (false, false, false);
using var fs = File.OpenRead(_filePath);
- foreach (var block in ReadBlocks(fs))
+ foreach (var block in ReadBlocks(fs, new ReadState()))
{
int n = block.Frames.Count;
return (n >= ColorFrames, n >= IntensityFrames, n >= NormalFrames);
@@ -618,8 +630,9 @@ private sealed record Block(double ScaleX, double ScaleY, double ScaleZ, double
///
///
/// Data that the reader cannot use is counted and reported in the warnings of the document: the points that were
- /// dropped as padding, the compressed streams that were skipped because they are too large and the point data
- /// that ends at a block the reader does not know. With the read fails with
+ /// dropped as padding, the compressed streams that were skipped because they are too large, the point blocks that
+ /// carry no positions or whose frame list ends before the block does, and the point data that ends at a block the
+ /// reader does not know. With the read fails with
/// instead of the warning.
///
///
@@ -635,14 +648,11 @@ public override IEnumerable ReadPoints(PclReadOptions? options = null)
int step = Math.Max(1, options.SubsampleStep);
long yielded = 0;
- // The counters describe this read; they start at zero every time.
- _droppedPoints = 0;
- _skippedStreams = 0;
- _blocksRead = 0;
- _chainEndedEarly = false;
+ // The counters describe this read alone; they start at zero every time.
+ var state = new ReadState();
using var fs = File.OpenRead(_filePath);
- foreach (var block in ReadBlocks(fs))
+ foreach (var block in ReadBlocks(fs, state))
{
var frames = block.Frames;
byte[] p0 = frames[0];
@@ -665,7 +675,7 @@ public override IEnumerable ReadPoints(PclReadOptions? options = null)
if (val == 0 && i > 0 && p0[i] == 0 && p1[i] == 0)
{
// The slot holds no point: it is padding or a point at the origin of the block.
- _droppedPoints++;
+ state.DroppedPoints++;
continue;
}
@@ -734,26 +744,32 @@ public override IEnumerable ReadPoints(PclReadOptions? options = null)
}
// The whole point data was read, so what it did not hold can be reported.
- ReportIncompleteData(options.Strict);
+ ReportIncompleteData(options.Strict, state);
}
///
/// Reports the data that the point data did not hold: the points that were dropped as padding, the compressed
- /// streams that were skipped and the point data that ends at a block the reader does not know. In strict mode
- /// the read fails instead.
+ /// streams that were skipped, the point blocks that carry no positions or whose frame list is cut short, and the
+ /// point data that ends at a block the reader does not know. In strict mode the read fails instead.
///
- private void ReportIncompleteData(bool strict)
+ private void ReportIncompleteData(bool strict, ReadState state)
{
var problems = new List();
- if (_droppedPoints > 0)
- problems.Add($"{_droppedPoints} points were dropped because their slot holds no point");
+ if (state.DroppedPoints > 0)
+ problems.Add($"{state.DroppedPoints} points were dropped because their slot holds no point");
+
+ if (state.SkippedStreams > 0)
+ problems.Add($"{state.SkippedStreams} compressed streams were skipped because they are larger than the accepted size");
+
+ if (state.UnusableBlocks > 0)
+ problems.Add($"{state.UnusableBlocks} point blocks were skipped because they hold fewer than four planes");
- if (_skippedStreams > 0)
- problems.Add($"{_skippedStreams} compressed streams were skipped because they are larger than the accepted size");
+ if (state.IncompleteBlocks > 0)
+ problems.Add($"{state.IncompleteBlocks} point blocks have a frame list that ends before the block does");
- if (_chainEndedEarly)
- problems.Add($"the point data ends after {_blocksRead} blocks at a block that is not a point block");
+ if (state.ChainEndedEarly)
+ problems.Add($"the point data ends after {state.ChainBlocks} blocks at a block that is not a point block");
if (problems.Count == 0) return;
@@ -764,7 +780,7 @@ private void ReportIncompleteData(bool strict)
}
/// Walks the chain of point blocks and yields those that carry point planes and belong to this scan.
- private IEnumerable ReadBlocks(FileStream fs)
+ private IEnumerable ReadBlocks(FileStream fs, ReadState state)
{
long fileLength = fs.Length;
long decodedLimit = DecodedAllowanceBytes + (fileLength * DecodedBytesPerFileByte);
@@ -808,7 +824,7 @@ private IEnumerable ReadBlocks(FileStream fs)
if (firstFrame >= 8)
{
long firstFrameOffset = blockStart + firstFrame;
- var frames = ReadAllFrames(fs, firstFrameOffset, blockEnd);
+ var frames = ReadAllFrames(fs, firstFrameOffset, blockEnd, state);
// Many small blocks that each expand a lot would otherwise make the file cost unbounded time.
foreach (byte[] plane in frames) decodedBytes += plane.Length;
@@ -823,18 +839,35 @@ private IEnumerable ReadBlocks(FileStream fs)
double.IsFinite(minX) && double.IsFinite(minY) && double.IsFinite(minZ)))
throw FormatError.Corrupted("point block has an invalid coordinate scale or origin.");
- _blocksRead++;
+ state.BlocksRead++;
yield return new Block(scaleX, scaleY, scaleZ, minX, minY, minZ, frames);
}
+ else state.UnusableBlocks++;
}
+ else state.UnusableBlocks++;
}
// Data that is left in the point data but that is not a point block means that the reader did not read
- // everything the container holds.
- if (chain.Count > 0)
+ // everything the container holds. The gap belongs to the payload, so only the unified scan or the scan
+ // that owns the last block reports it, and the count is the count of the whole chain.
+ state.ChainBlocks = chain.Count;
+ if (_blockStarts == null || (chain.Count > 0 && BelongsToScan(chain[^1].Start)))
{
- long lastEnd = chain[^1].Start + chain[^1].Length;
- if (payloadEnd - lastEnd >= 8 + 128) _chainEndedEarly = true;
+ if (chain.Count > 0)
+ {
+ long lastEnd = chain[^1].Start + chain[^1].Length;
+ if (payloadEnd - lastEnd >= 8 + 128) state.ChainEndedEarly = true;
+ }
+ else
+ {
+ // An empty chain means the very first chunk is not a point block. When a full block could have
+ // fit there, the point data ends at a block the reader does not know, just as with a later one.
+ fs.Position = (long)_entry!.Offset;
+ byte[] payloadHeader = new byte[64];
+ fs.ReadExactly(payloadHeader);
+ long firstBlock = (long)_entry.Offset + Bytes.ReadUInt32LE(payloadHeader.AsSpan(0x20, 4));
+ if (payloadEnd - firstBlock >= 8 + 128) state.ChainEndedEarly = true;
+ }
}
}
@@ -842,7 +875,7 @@ private IEnumerable ReadBlocks(FileStream fs)
private bool BelongsToScan(long blockStart) =>
_blockStarts == null || Array.BinarySearch(_blockStarts, blockStart) >= 0;
- private List ReadAllFrames(FileStream fs, long offset, long maxOffset)
+ private List ReadAllFrames(FileStream fs, long offset, long maxOffset, ReadState state)
{
var list = new List();
long limit = Math.Min(maxOffset, fs.Length);
@@ -863,16 +896,25 @@ private List ReadAllFrames(FileStream fs, long offset, long maxOffset)
// A stream that is larger than the accepted size is skipped: the rest of the block is lost with it.
if (fLen > MaxCompressedFrameBytes)
{
- _skippedStreams++;
+ state.SkippedStreams++;
break;
}
- if ((long)fLen > limit - fs.Position) break;
+ // A frame list that ends before the block does leaves the rest of the block unread.
+ if ((long)fLen > limit - fs.Position)
+ {
+ state.IncompleteBlocks++;
+ break;
+ }
byte[] fData = new byte[(int)fLen];
fs.ReadExactly(fData);
- if (!fData.AsSpan().StartsWith(FrameMagic)) break;
+ if (!fData.AsSpan().StartsWith(FrameMagic))
+ {
+ state.IncompleteBlocks++;
+ break;
+ }
if (list.Count >= MaxFramesPerBlock)
throw FormatError.Corrupted($"point block has more than {MaxFramesPerBlock} planes.");
diff --git a/tests/PclIO.Tests/FormatTests.cs b/tests/PclIO.Tests/FormatTests.cs
index 62b5a13..bb42511 100644
--- a/tests/PclIO.Tests/FormatTests.cs
+++ b/tests/PclIO.Tests/FormatTests.cs
@@ -547,6 +547,57 @@ public void A_Lgsx_Container_Reports_Point_Data_That_Ends_At_A_Block_It_Does_Not
Assert.Contains(doc.Warnings, w => w.Contains("ends after 1 blocks", StringComparison.Ordinal));
}
+ [Fact]
+ public void A_Lgsx_Container_Whose_First_Chunk_Is_Not_A_Point_Block_Reports_It()
+ {
+ // The chain is empty because the very first chunk is unknown; the point data still ends at a block the
+ // reader does not know, and the declared count is not what a read returns.
+ using var dir = new TempDirectory();
+ using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(unknownFirstBlock: true)));
+
+ var scan = Assert.Single(doc.Scans);
+ Assert.Equal(5, scan.PointCount);
+ Assert.Empty(scan.ReadPoints());
+ Assert.Contains(doc.Warnings, w => w.Contains("ends after 0 blocks", StringComparison.Ordinal));
+ Assert.Throws(() => scan.ReadPoints(new PclReadOptions { Strict = true }).ToList());
+ }
+
+ [Fact]
+ public void A_Lgsx_Block_Whose_Frame_List_Is_Cut_Short_Is_Reported()
+ {
+ using var dir = new TempDirectory();
+ using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(truncatedFrameList: true)));
+
+ var scan = Assert.Single(doc.Scans);
+ Assert.Equal(5, scan.ReadPoints().Count());
+ Assert.Contains(doc.Warnings, w => w.Contains("1 point blocks have a frame list that ends before the block does", StringComparison.Ordinal));
+ }
+
+ [Fact]
+ public void A_Lgsx_Block_With_Fewer_Than_Four_Planes_Is_Reported()
+ {
+ using var dir = new TempDirectory();
+ using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(fewPlanes: true)));
+
+ var scan = Assert.Single(doc.Scans);
+ Assert.Empty(scan.ReadPoints());
+ Assert.Contains(doc.Warnings, w => w.Contains("1 point blocks were skipped because they hold fewer than four planes", StringComparison.Ordinal));
+ }
+
+ [Fact]
+ public void Interleaved_Reads_Of_A_Lgsx_Scan_Keep_Their_Own_Counts()
+ {
+ // Two enumerators advance side by side; the counters of one read must not reset or mix with the other's.
+ var points = new List { new(1, 2, 3), new(0, 0, 0), new(4, 5, 6), new(0, 0, 0) };
+ using var dir = new TempDirectory();
+ using var doc = LgsxDocument.Open(dir.Write("site.lgsx", SyntheticFixtures.CreateSyntheticLgsx(points)));
+
+ var scan = Assert.Single(doc.Scans);
+ var pairs = scan.ReadPoints().Zip(scan.ReadPoints(), (a, b) => (a, b)).ToList();
+ Assert.Equal(2, pairs.Count);
+ Assert.Contains(doc.Warnings, w => w.Contains("2 points were dropped", StringComparison.Ordinal));
+ }
+
[Fact]
public void A_Lgsx_Container_Fails_In_Strict_Mode_When_Its_Point_Data_Is_Incomplete()
{
diff --git a/tests/PclIO.Tests/SyntheticFixtures.cs b/tests/PclIO.Tests/SyntheticFixtures.cs
index 9b5ced0..2509a4e 100644
--- a/tests/PclIO.Tests/SyntheticFixtures.cs
+++ b/tests/PclIO.Tests/SyntheticFixtures.cs
@@ -157,10 +157,13 @@ public static byte[] CreateZip(string entryName, string text)
/// False leaves the number of points out of the header of the point data.
/// True writes a frame whose declared length is above the accepted size.
/// True writes a block after the point blocks that the reader does not know.
+ /// True writes a chunk the reader does not know before the point blocks.
+ /// True ends the frame list of a block with a length prefix that declares more bytes than the block holds.
+ /// True writes the blocks with fewer than four planes, so they carry no positions.
public static byte[] CreateSyntheticLgsx(IReadOnlyList? points = null, bool withColorAndIntensity = false, bool withPointData = true,
IReadOnlyList? extraPlanes = null, int setupCount = 1, bool assignPointsToSetups = false, bool overlappingSetupRanges = false,
bool collidingSetupNames = false, bool damagedSetupRecord = false, bool damagedPointData = false, bool withPointCount = true,
- bool oversizedFrame = false, bool unknownBlock = false)
+ bool oversizedFrame = false, bool unknownBlock = false, bool unknownFirstBlock = false, bool truncatedFrameList = false, bool fewPlanes = false)
{
points ??= DefaultLeicaPoints;
using var ms = new MemoryStream();
@@ -174,7 +177,8 @@ public static byte[] CreateSyntheticLgsx(IReadOnlyList? points
: new List> { points };
(byte[] payload, List<(long Start, long Length)> blocks) =
- BuildLeicaPayload(groups, withColorAndIntensity, extraPlanes ?? [], damagedPointData, withPointCount, oversizedFrame, unknownBlock);
+ BuildLeicaPayload(groups, withColorAndIntensity, extraPlanes ?? [], damagedPointData, withPointCount, oversizedFrame, unknownBlock,
+ unknownFirstBlock, truncatedFrameList, fewPlanes);
// The point data follows the container entries and their records, so its start is known before they are written.
var setupNames = Enumerable.Range(0, setupCount).Select(i => "SETUP-" + (3 + i)).ToList();
@@ -302,7 +306,8 @@ private static void WriteEntry(BinaryWriter bw, string name, ulong payloadOffset
// Builds the point data payload: a header, then one block per group of points.
private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLeicaPayload(
IReadOnlyList> groups, bool full, IReadOnlyList extraPlanes, bool damaged = false,
- bool withPointCount = true, bool oversizedFrame = false, bool unknownBlock = false)
+ bool withPointCount = true, bool oversizedFrame = false, bool unknownBlock = false,
+ bool unknownFirstBlock = false, bool truncatedFrameList = false, bool fewPlanes = false)
{
using var ms = new MemoryStream();
using var bw = new BinaryWriter(ms);
@@ -315,6 +320,14 @@ private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLei
BinaryPrimitives.WriteUInt64LittleEndian(header.AsSpan(0x28, 8), (ulong)groups.Sum(g => g.Count));
bw.Write(header);
+ // A chunk that is not a point block before the point blocks: the chain stays empty.
+ if (unknownFirstBlock)
+ {
+ bw.Write("XPRB"u8.ToArray());
+ bw.Write((uint)(8 + 128));
+ bw.Write(new byte[128]);
+ }
+
var blocks = new List<(long Start, long Length)>();
foreach (var group in groups)
{
@@ -322,6 +335,7 @@ private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLei
if (group.Count == 0) continue;
var frames = BuildFrames(group, full, extraPlanes);
+ if (fewPlanes) frames = [.. frames.Take(3)];
if (damaged)
{
// A frame whose descriptor is not valid: the header of the block stays readable, the data does not.
@@ -330,7 +344,7 @@ private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLei
// A frame whose declared length is above the accepted size is skipped by the reader.
const ulong OversizedLength = 20_000_000;
- int blockLength = 8 + 128 + frames.Sum(f => 8 + f.Length) + (oversizedFrame ? 8 : 0);
+ int blockLength = 8 + 128 + frames.Sum(f => 8 + f.Length) + (oversizedFrame ? 8 : 0) + (truncatedFrameList ? 8 : 0);
long start = ms.Position;
bw.Write("APRN"u8.ToArray());
@@ -348,6 +362,9 @@ private static (byte[] Payload, List<(long Start, long Length)> Blocks) BuildLei
if (oversizedFrame) bw.Write(OversizedLength);
+ // A length prefix that declares more bytes than the block holds: the frame list is cut short.
+ if (truncatedFrameList) bw.Write((ulong)1000);
+
blocks.Add((start, blockLength));
}