diff --git a/CHANGELOG.md b/CHANGELOG.md index e373d46..870f0e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `PclReadOptions.ExcludeFilteredPoints` excludes the points with a non-zero flag; by default every stored point is returned. - The normals (`Normals`) of the PinPoint sectioned layout are read when the file holds them, and `HasNormals` reports what the file contains. +- The PinPoint coordinate plausibility rule is now `PclReadOptions.MaxCoordinateMagnitude` (default 5000, null for no + limit); non-finite coordinates are still never returned. A full read reports how many points it dropped in the warnings + of the document, and fails instead with `PclReadOptions.Strict`. ## [0.1.0] - 2026-11-15 diff --git a/README.md b/README.md index 5dc6c11..0852843 100644 --- a/README.md +++ b/README.md @@ -87,8 +87,10 @@ These readers drop data without an error or a warning, so a scan can yield fewer scan is the number of the cells that are read, so it matches what a full read returns. A scan folder whose data stream does not hold a whole number of samples for the grid is read with the assumed layout of eight bytes per sample, and a warning says so. -- PinPoint: a point with a NaN or infinite coordinate, or a coordinate whose magnitude exceeds 5000 in the file's own unit, is not - returned. This is a plausibility rule of the reader, so coordinates in a projected system with large offsets are dropped. A +- PinPoint: a point with a NaN or infinite coordinate is never returned, and neither is a point with a coordinate whose + magnitude exceeds `PclReadOptions.MaxCoordinateMagnitude` in the file's own unit (5000 by default, null for no limit). + This is a plausibility rule of the reader, so coordinates in a projected system with large offsets are dropped by default. + A full read reports how many points it dropped in the warnings of the document, and fails instead with `PclReadOptions.Strict`. A data file in neither layout, or in the sectioned layout with an element type or header value the reader does not support, cannot be read: the project lists it with a warning and reading its points throws `NotSupportedException`. A compressed stream whose copies reach back more than 1 MiB is not decoded either: reading its points throws `NotSupportedException`, without an earlier warning. diff --git a/docs/getting-started.md b/docs/getting-started.md index ec73acd..9ecad39 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -99,9 +99,10 @@ file that exceeds a limit is rejected with `InvalidDataException`, with the exce ## Limitations - The API can change before 1.0. -- Some readers drop points that fail a plausibility rule without a warning: FARO cells outside 0.1 m to 350 m, PinPoint points with a - non-finite coordinate or a coordinate beyond 5000 in the file's own unit, and Leica padding entries. A scan can therefore yield fewer - points than `PointCount`. The rules and the limits on file sizes and counts are listed in the repository README. +- Some readers drop points that fail a plausibility rule: FARO cells outside 0.1 m to 350 m, PinPoint points with a + non-finite coordinate or a coordinate beyond `PclReadOptions.MaxCoordinateMagnitude` in the file's own unit (5000 by default), + and Leica padding entries. A scan can therefore yield fewer points than `PointCount`. The rules and the limits on file sizes + and counts are listed in the repository README. - `SubsampleStep` is applied inside each format's internal blocks, so the spacing is approximate; `MaxPoints` limits one read call. - LAS and PTS export read the source twice; memory does not depend on the number of points. A point without intensity is written with full intensity and a point without colour as white. LAS uses a 1 mm scale, so the extent of each axis must not exceed about 2147 km. diff --git a/docs/public-api.md b/docs/public-api.md index c471be9..d15bfea 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -15,7 +15,7 @@ format-specific members can change when the support for that format changes. |---|---| | `PclFile` | Static entry point. `DetectKind(path)` and `DetectKind(stream, extension)` return a `PclFileKind`; `Open(path, options)` detects the format and returns a `PclDocument`. | | `PclFileKind` | `Unknown`, `E57`, `RecapProject`, `RecapScan`, `LeicaLgsx`, `LeicaLgs`, `FaroFls`, `PinPoint` and `EdgeWise`. | -| `PclReadOptions` | `MaxPoints` (limit of one read call, null for all, 0 for none), `SubsampleStep` (1 keeps every point, N keeps about one in N), `ApplyTransform` (default true), `Strict` (default false: a reader that finds incomplete data fails with `InvalidDataException` instead of adding a warning) and `ExcludeFilteredPoints` (default false: points the file marks as filtered are excluded; currently the `FilterBits` channel of the PinPoint sectioned layout). Negative `MaxPoints` and a step below 1 throw `ArgumentOutOfRangeException`. `PclReadOptions.Default` has no limit. | +| `PclReadOptions` | `MaxPoints` (limit of one read call, null for all, 0 for none), `MaxCoordinateMagnitude` (default 5000, null for no limit; NaN and infinite coordinates are never returned; currently only PinPoint applies it), `SubsampleStep` (1 keeps every point, N keeps about one in N), `ApplyTransform` (default true), `Strict` (default false: a reader that finds incomplete data fails with `InvalidDataException` instead of adding a warning) and `ExcludeFilteredPoints` (default false: points the file marks as filtered are excluded; currently the `FilterBits` channel of the PinPoint sectioned layout). Negative `MaxPoints`, a negative magnitude limit and a step below 1 throw `ArgumentOutOfRangeException`. `PclReadOptions.Default` has no limit. | `Open` throws `FileNotFoundException` for a missing path, `DirectoryNotFoundException` when a directory the reader needs is missing or disappears while reading, `NotSupportedException` for an unrecognised format or an unsupported feature, and diff --git a/src/PclIO/Formats/PinPoint/PinPointDocument.cs b/src/PclIO/Formats/PinPoint/PinPointDocument.cs index 2aada22..483db74 100644 --- a/src/PclIO/Formats/PinPoint/PinPointDocument.cs +++ b/src/PclIO/Formats/PinPoint/PinPointDocument.cs @@ -34,6 +34,15 @@ private PinPointDocument(string path, long fileSize, string projectName) _metadata["ProjectName"] = projectName; } + /// + /// Adds a warning to the document. The same message is added only once, so that reading the same scan twice + /// does not repeat it. + /// + internal void AddWarning(string message) + { + if (!_warnings.Contains(message, StringComparer.Ordinal)) _warnings.Add(message); + } + /// Opens a PinPoint project from a manifest file, a .srf file with its data folder, or a project folder. /// /// Manifest (.smf, ScanProject.json), .srf file or project directory. When a directory holds several candidates and no @@ -134,7 +143,7 @@ public static PinPointDocument Open(string path, PclReadOptions? options = null) if (binPath == null) document._warnings.Add($"Scan '{name}': point data file not found."); - document.AddScan(new PinPointScan(hash, name, pointCount, transform, binPath, table)); + document.AddScan(new PinPointScan(hash, name, pointCount, transform, binPath, table, document)); } } else if (TryProp(root, "DataObjects", out var dataObjs) && dataObjs.ValueKind == JsonValueKind.Array) @@ -164,7 +173,7 @@ public static PinPointDocument Open(string path, PclReadOptions? options = null) document._warnings.Add($"Location '{locHash}': point data file not found."); var table = ReadTableOrWarn(document, locHash, binPath); - document.AddScan(new PinPointScan(locHash, locHash, 0, transform, binPath, table)); + document.AddScan(new PinPointScan(locHash, locHash, 0, transform, binPath, table, document)); } } } @@ -409,10 +418,8 @@ private static void Assign(int tag, long length, long dataStart, out long offset /// Scan representation in a PinPoint project. public sealed class PinPointScan : PclScan { - // Coordinates with a larger magnitude (metres) are dropped; see ReadPoints for what this means. - private const float MaxCoordinate = 5000.0f; - private readonly PinPointDocument.ChannelTable? _table; + private readonly PinPointDocument? _document; /// public override string Id { get; } @@ -439,7 +446,7 @@ public sealed class PinPointScan : PclScan public bool HasFilterBits => _table is { HasFilterBits: true }; internal PinPointScan(string id, string name, long pointCount, PclTransform transform, string? binFilePath, - PinPointDocument.ChannelTable? table) + PinPointDocument.ChannelTable? table, PinPointDocument? document) { Id = id; Name = name; @@ -447,17 +454,20 @@ internal PinPointScan(string id, string name, long pointCount, PclTransform tran Transform = transform; BinFilePath = binFilePath; _table = table; + _document = document; } /// Streams points from the point data file; yields nothing when the file was not found. /// /// - /// Points are positions as float32 triples. Points with a NaN or infinite coordinate, or with any coordinate whose - /// magnitude exceeds 5000 in the file's own unit, are not returned and - /// are not counted; the remaining points keep consecutive values. The magnitude limit - /// is a plausibility rule of this reader, not a property of the file format, so positions in a projected coordinate - /// system with large offsets are dropped. reports the manifest or channel-table count, so - /// it can be larger than the number of returned points. + /// Points are positions as float32 triples. Points with a NaN or infinite coordinate are never returned. A point + /// with a coordinate whose magnitude exceeds in the file's own + /// unit is not returned either, unless the limit is switched off; the default of 5000 is a plausibility rule of + /// this reader, not a property of the file format, so positions in a projected coordinate system with large offsets + /// are dropped by default. Dropped points are not counted; the remaining points keep consecutive + /// values, and the document gains a warning that tells how many were dropped. + /// reports the manifest or channel-table count, so it can be larger than the number of + /// returned points. /// /// /// A point data file whose layout is neither a table of named channels nor a list of sections with a property table is @@ -503,6 +513,13 @@ private IEnumerable ReadChannels(PinPointDocument.ChannelTable table, // A point whose flag byte is not zero is marked as filtered; it is only skipped when the caller asks for it. bool skipFiltered = options.ExcludeFilteredPoints && table.HasFilterBits; + // Coordinates that are NaN or infinite are never returned; coordinates beyond the magnitude limit are not + // returned either, unless the caller switched the limit off. + double? magnitudeLimit = options.MaxCoordinateMagnitude; + + long droppedNonFinite = 0; + long droppedOutOfRange = 0; + int colorSize = table.ColorSize; long pointTotal = table.PointCount; long colorTotal = !table.HasColors ? 0 : table.Packed ? pointTotal : table.ColorsLength / colorSize; @@ -523,6 +540,7 @@ private IEnumerable ReadChannels(PinPointDocument.ChannelTable table, long yielded = 0; long start = 0; + long visited = 0; for (; start < pointTotal && yielded < targetCount; start += Batch) { int count = (int)Math.Min(Batch, pointTotal - start); @@ -539,6 +557,7 @@ private IEnumerable ReadChannels(PinPointDocument.ChannelTable table, for (int j = 0; j < count && yielded < targetCount; j++) { + visited++; if ((start + j) % step != 0) continue; if (filters != null && filterBytes[j] != 0) continue; @@ -546,7 +565,18 @@ private IEnumerable ReadChannels(PinPointDocument.ChannelTable table, float x = Bytes.ReadSingleLE(record[..4]); float y = Bytes.ReadSingleLE(record.Slice(4, 4)); float z = Bytes.ReadSingleLE(record.Slice(8, 4)); - if (!IsPlausible(x, y, z)) continue; + if (!float.IsFinite(x) || !float.IsFinite(y) || !float.IsFinite(z)) + { + droppedNonFinite++; + continue; + } + + if (magnitudeLimit != null && + (Math.Abs(x) > magnitudeLimit || Math.Abs(y) > magnitudeLimit || Math.Abs(z) > magnitudeLimit)) + { + droppedOutOfRange++; + continue; + } PclColor? color = null; if (j < colorCount) @@ -593,6 +623,29 @@ private IEnumerable ReadChannels(PinPointDocument.ChannelTable table, foreach (var stream in new[] { points, colors, normals, intensities, filters }) if (stream != null && stream.ReadByte() >= 0) throw FormatError.Corrupted("a compressed channel holds more data than it declares"); } + + // The report describes the whole point data. A read that MaxPoints stopped early visited only a part of it, + // so its counts say nothing about the rest and are not reported. + if (visited >= pointTotal) + ReportDroppedPoints(options, magnitudeLimit, droppedNonFinite, droppedOutOfRange); + } + + /// + /// Reports the points that the plausibility rule dropped: a warning tells how many had non-finite coordinates and + /// how many were beyond the magnitude limit. In strict mode the read fails instead. Called only when the whole + /// point data was read, so the counts describe all of it. + /// + private void ReportDroppedPoints(PclReadOptions options, double? magnitudeLimit, long nonFinite, long outOfRange) + { + if (nonFinite == 0 && outOfRange == 0) return; + + string message = magnitudeLimit == null + ? $"The coordinate plausibility rule dropped {nonFinite} points with non-finite coordinates." + : $"The coordinate plausibility rule dropped {nonFinite + outOfRange} points " + + $"({nonFinite} with non-finite coordinates, {outOfRange} beyond the magnitude limit of {magnitudeLimit})."; + + if (options.Strict) throw FormatError.Corrupted(message); + _document?.AddWarning(message); } // A stream over one channel: the bytes of the file at the offset, or their Snappy decoding for the packed form. @@ -612,9 +665,4 @@ private IEnumerable ReadChannels(PinPointDocument.ChannelTable table, throw; } } - - // Finite coordinates of at most MaxCoordinate in magnitude; everything else is dropped by the callers. - private static bool IsPlausible(float x, float y, float z) => - float.IsFinite(x) && float.IsFinite(y) && float.IsFinite(z) && - Math.Abs(x) <= MaxCoordinate && Math.Abs(y) <= MaxCoordinate && Math.Abs(z) <= MaxCoordinate; } diff --git a/src/PclIO/PclReadOptions.cs b/src/PclIO/PclReadOptions.cs index 5569201..786546e 100644 --- a/src/PclIO/PclReadOptions.cs +++ b/src/PclIO/PclReadOptions.cs @@ -24,6 +24,26 @@ public long? MaxPoints } } + private readonly double? _maxCoordinateMagnitude = 5000.0; + + /// + /// Maximum allowed magnitude of a point coordinate in the file's own unit; a point with a larger coordinate is not + /// returned. Null means no magnitude limit. Coordinates that are NaN or infinite are never returned. Currently only + /// the PinPoint reader applies this rule; other readers ignore this option. The default is 5000. + /// + /// The value is negative or not finite. + public double? MaxCoordinateMagnitude + { + get => _maxCoordinateMagnitude; + init + { + ArgumentOutOfRangeException.ThrowIfNegative(value ?? 0, nameof(value)); + if (value is double v && !double.IsFinite(v)) + throw new ArgumentOutOfRangeException(nameof(value), "The magnitude limit must be finite; null means no limit."); + _maxCoordinateMagnitude = value; + } + } + /// /// Subsampling step: 1 keeps every point, N keeps one point in N. Readers apply the step inside their own /// internal blocks (packets, chunks, clusters), so the spacing is approximate and depends on the format's block layout. diff --git a/tests/PclIO.Tests/FormatTests.cs b/tests/PclIO.Tests/FormatTests.cs index 99f983d..b4180cd 100644 --- a/tests/PclIO.Tests/FormatTests.cs +++ b/tests/PclIO.Tests/FormatTests.cs @@ -1028,6 +1028,65 @@ public void PinPoint_Points_Beyond_A_Plausible_Range_Are_Not_Reported() Assert.Equal(new PclVector3(1, 2, 3), point.Position); } + [Fact] + public void PinPoint_Magnitude_Limit_Can_Be_Raised_Or_Switched_Off() + { + using var dir = new TempDirectory(); + byte[] bin = SyntheticFixtures.CreateSyntheticPinPointBin([(1, 2, 3), (1e9f, 0, 0), (float.NaN, 0, 0)]); + using var doc = PinPointDocument.Open(SyntheticFixtures.WritePinPointProject(dir, bin)); + + // A raised limit keeps the large coordinate; a missing limit keeps it too, but never the NaN. + var raised = doc.Scans[0].ReadPoints(new PclReadOptions { ApplyTransform = false, MaxCoordinateMagnitude = 2e9 }).ToList(); + Assert.Equal(2, raised.Count); + Assert.Equal(new PclVector3(1e9f, 0, 0), raised[1].Position); + + var unlimited = doc.Scans[0].ReadPoints(new PclReadOptions { ApplyTransform = false, MaxCoordinateMagnitude = null }).ToList(); + Assert.Equal(2, unlimited.Count); + } + + [Fact] + public void PinPoint_Dropped_Points_Are_Reported_With_A_Count() + { + using var dir = new TempDirectory(); + byte[] bin = SyntheticFixtures.CreateSyntheticPinPointBin([(1, 2, 3), (1e9f, 0, 0), (float.NaN, 0, 0)]); + using var doc = PinPointDocument.Open(SyntheticFixtures.WritePinPointProject(dir, bin)); + + Assert.Single(doc.Scans[0].ReadPoints(NoTransform)); + Assert.Contains(doc.Warnings, w => w.Contains("dropped 2 points", StringComparison.Ordinal)); + } + + [Fact] + public void PinPoint_Dropped_Points_Fail_In_Strict_Mode() + { + using var dir = new TempDirectory(); + byte[] bin = SyntheticFixtures.CreateSyntheticPinPointBin([(1, 2, 3), (1e9f, 0, 0)]); + using var doc = PinPointDocument.Open(SyntheticFixtures.WritePinPointProject(dir, bin)); + + Assert.Throws(() => doc.Scans[0].ReadPoints(new PclReadOptions { Strict = true }).ToList()); + } + + [Fact] + public void PinPoint_A_Read_Stopped_By_MaxPoints_Reports_Nothing() + { + // The cap stops the read after a part of the point data; the counts of that part say nothing about the + // rest, so neither the warning nor the strict failure fires. + using var dir = new TempDirectory(); + byte[] bin = SyntheticFixtures.CreateSyntheticPinPointBin([(float.NaN, 0, 0), (1, 2, 3), (1e9f, 0, 0)]); + using var doc = PinPointDocument.Open(SyntheticFixtures.WritePinPointProject(dir, bin)); + + var scan = doc.Scans[0]; + var capped = new PclReadOptions { ApplyTransform = false, MaxPoints = 1 }; + var point = Assert.Single(scan.ReadPoints(capped)); + Assert.Equal(new PclVector3(1, 2, 3), point.Position); + Assert.DoesNotContain(doc.Warnings, w => w.Contains("dropped", StringComparison.Ordinal)); + + Assert.Single(scan.ReadPoints(capped with { Strict = true })); + Assert.DoesNotContain(doc.Warnings, w => w.Contains("dropped", StringComparison.Ordinal)); + + // A full read of the same data still reports the drops. + Assert.Throws(() => scan.ReadPoints(new PclReadOptions { ApplyTransform = false, Strict = true }).ToList()); + } + // ---- EdgeWise -------------------------------------------------------------------------------------------- private static readonly C3Cluster[] Clusters = diff --git a/tests/PclIO.Tests/OptionsAndPathTests.cs b/tests/PclIO.Tests/OptionsAndPathTests.cs index 562dc5d..2e0d365 100644 --- a/tests/PclIO.Tests/OptionsAndPathTests.cs +++ b/tests/PclIO.Tests/OptionsAndPathTests.cs @@ -13,8 +13,11 @@ public void Default_Options_Read_Everything_And_Apply_The_Transform() var options = new PclReadOptions(); Assert.Null(options.MaxPoints); + Assert.Equal(5000.0, options.MaxCoordinateMagnitude); Assert.Equal(1, options.SubsampleStep); Assert.True(options.ApplyTransform); + Assert.False(options.Strict); + Assert.False(options.ExcludeFilteredPoints); Assert.Equal(options, PclReadOptions.Default); } @@ -45,6 +48,23 @@ public void A_Subsample_Step_Of_One_Or_More_Is_Accepted() Assert.Equal(int.MaxValue, new PclReadOptions { SubsampleStep = int.MaxValue }.SubsampleStep); } + [Theory] + [InlineData(-1.0)] + [InlineData(double.MinValue)] + [InlineData(double.NaN)] + [InlineData(double.PositiveInfinity)] + [InlineData(double.NegativeInfinity)] + public void A_Negative_Or_Non_Finite_Magnitude_Limit_Is_Rejected(double value) => + Assert.Throws(() => new PclReadOptions { MaxCoordinateMagnitude = value }); + + [Fact] + public void A_Magnitude_Limit_Of_Zero_Or_More_Is_Accepted() + { + Assert.Equal(0, new PclReadOptions { MaxCoordinateMagnitude = 0 }.MaxCoordinateMagnitude); + Assert.Null(new PclReadOptions { MaxCoordinateMagnitude = null }.MaxCoordinateMagnitude); + Assert.Equal(1e9, new PclReadOptions { MaxCoordinateMagnitude = 1e9 }.MaxCoordinateMagnitude); + } + [Fact] public void A_Limit_Applies_To_The_Total_Across_All_Scans() {