diff --git a/src/TdbIO/Internal/Nc1Reader.cs b/src/TdbIO/Internal/Nc1Reader.cs index 3bfbab1..501ec0c 100644 --- a/src/TdbIO/Internal/Nc1Reader.cs +++ b/src/TdbIO/Internal/Nc1Reader.cs @@ -24,48 +24,81 @@ public static Nc1Document Read(ReadOnlySpan data, List warnings) throw FormatError.At("DSTV file is larger than the 16 MiB limit", MaxFileLength); if (!FirstLineIsSt(data)) throw FormatError.At("not a DSTV file: line 1 is not ST", 0, 1); - var lines = SplitLines(data); - var (comment, fields) = ReadStBlock(lines, data.Length); + var reader = new LineReader(data); + reader.ReadLine("DSTV file ends inside the ST block", data.Length); + + var (comment, fields, length, stRaw) = ReadStBlock(ref reader, data.Length); var blocks = new List { - new("ST", 1, lines.Skip(1).Take(StLines - 1).Select(l => l.Text).ToArray(), []), + new("ST", 1, stRaw, []), }; - int at = StLines; - while (at < lines.Count) + Line head = reader.ReadLine("DSTV file ends inside the ST block", data.Length); + if (!IsCode(head.Text)) + throw Error("the ST block is not followed by a block code", head); + + while (head.Text is not null) { - Line head = lines[at]; - int end = at + 1; - while (end < lines.Count && !IsCode(lines[end].Text)) end++; - var body = lines.Skip(at + 1).Take(end - at - 1).ToArray(); - var raw = body.Select(l => l.Text).ToArray(); - switch (head.Text) + if (head.Text == "ST") + throw Error("an ST block may only start the file", head); + + var raw = new List(); + var lines = new List(); + Line nextHead = default; + + while (reader.TryReadLine(out Line line)) { - case "ST": - throw Error("an ST block may only start the file", head); - case "EN": - if (body.Length > 0) throw Error("data after the EN block", body[0]); - if (end < lines.Count) throw Error("data after the EN block", lines[end]); - blocks.Add(new("EN", head.Number, raw, [])); - break; - case "AK": - blocks.Add(new("AK", head.Number, raw, ReadAk(head, body, warnings))); - break; - case "BO": - blocks.Add(new("BO", head.Number, raw, ReadBo(body))); - break; - default: - // Other codes are not read; their lines are kept as written. - blocks.Add(new(head.Text, head.Number, raw, [])); + if (IsCode(line.Text)) + { + nextHead = line; break; + } + + switch (head.Text) + { + case "EN": + throw Error("data after the EN block", line); + case "AK": + lines.Add(ValidateAndParseAkLine(line, lines.Count)); + break; + case "BO": + lines.Add(ValidateAndParseBoLine(line)); + break; + } + raw.Add(line.Text); + } + + if (head.Text == "EN") + { + if (nextHead.Text is not null) + throw Error("data after the EN block", nextHead); + blocks.Add(new("EN", head.Number, raw.ToArray(), [])); } - at = end; + else if (head.Text == "AK") + { + if (lines.Count == 0) throw Error("AK block has no lines", head); + var first = lines[0].Values; + var last = lines[^1].Values; + if (first[0].Number != last[0].Number || first[1].Number != last[1].Number) + warnings.Add($"AK block at line {head.Number}: the last point differs from the first, the contour is open"); + blocks.Add(new("AK", head.Number, raw.ToArray(), lines.ToArray())); + } + else if (head.Text == "BO") + { + blocks.Add(new("BO", head.Number, raw.ToArray(), lines.ToArray())); + } + else + { + blocks.Add(new(head.Text, head.Number, raw.ToArray(), [])); + } + + head = nextHead; } + if (blocks[^1].Code != "EN") - throw FormatError.At("DSTV file is cut: the last block is not EN", data.Length, lines.Count + 1); + throw FormatError.At("DSTV file is cut: the last block is not EN", data.Length, reader.LineNumber + 1); int? quantity = int.TryParse(fields[5], NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out int q) ? q : null; - decimal length = ParseDecimal(fields[8], lines[2 + 8]); return new Nc1Document(comment, fields, quantity, length, blocks, warnings); } @@ -75,48 +108,75 @@ private static bool FirstLineIsSt(ReadOnlySpan data) return (end < 0 ? data : data[..end]).SequenceEqual("ST"u8); } - /// Splits on CR LF only. Every byte is checked: ASCII, no control bytes, no lone CR or LF, and the last line - /// must end with CR LF. - private static List SplitLines(ReadOnlySpan data) + private ref struct LineReader { - var lines = new List(); - int start = 0; - for (int i = 0; i < data.Length; i++) + private readonly ReadOnlySpan _data; + private int _position; + private int _lineNumber; + + public LineReader(ReadOnlySpan data) + { + _data = data; + _position = 0; + _lineNumber = 0; + } + + public int LineNumber => _lineNumber; + + public bool TryReadLine(out Line line) { - byte b = data[i]; - int number = lines.Count + 1; - if (b == '\r') + if (_position >= _data.Length) { - if (i + 1 >= data.Length || data[i + 1] != '\n') - throw FormatError.At("CR not followed by LF", i, number); - lines.Add(new(Encoding.ASCII.GetString(data[start..i]), start, number)); - i++; - start = i + 1; - continue; + line = default; + return false; } - if (b == '\n') throw FormatError.At("LF without CR", i, number); - if (b >= 0x80) throw FormatError.At($"byte 0x{b:X2} is not ASCII", i, number); - if (b < 0x20 || b == 0x7F) throw FormatError.At($"control byte 0x{b:X2}", i, number); - if (i - start >= MaxLineLength) - throw FormatError.At("line is longer than the 4096 character limit", start, number); + + int start = _position; + int number = ++_lineNumber; + + for (int i = start; i < _data.Length; i++) + { + byte b = _data[i]; + if (b == '\r') + { + if (i + 1 >= _data.Length || _data[i + 1] != '\n') + throw FormatError.At("CR not followed by LF", i, number); + line = new Line(Encoding.ASCII.GetString(_data[start..i]), start, number); + _position = i + 2; + return true; + } + if (b == '\n') throw FormatError.At("LF without CR", i, number); + if (b >= 0x80) throw FormatError.At($"byte 0x{b:X2} is not ASCII", i, number); + if (b < 0x20 || b == 0x7F) throw FormatError.At($"control byte 0x{b:X2}", i, number); + if (i - start >= MaxLineLength) + throw FormatError.At("line is longer than the 4096 character limit", start, number); + } + + throw FormatError.At("DSTV file is cut: the last line does not end with CR LF", start, number); + } + + public Line ReadLine(string eofMessage, long eofOffset) + { + if (!TryReadLine(out Line line)) + throw FormatError.At(eofMessage, eofOffset, _lineNumber + 1); + return line; } - if (start != data.Length) - throw FormatError.At("DSTV file is cut: the last line does not end with CR LF", start, lines.Count + 1); - return lines; } - private static (string Comment, string[] Fields) ReadStBlock(List lines, long length) + private static (string Comment, string[] Fields, decimal Length, string[] RawLines) ReadStBlock(ref LineReader reader, long length) { - if (lines.Count <= StLines) - throw FormatError.At("DSTV file ends inside the ST block", length, lines.Count + 1); + var rawLines = new string[StLines - 1]; - Line second = lines[1]; + Line second = reader.ReadLine("DSTV file ends inside the ST block", length); if (!second.Text.StartsWith("** ", StringComparison.Ordinal)) throw Error("line 2 does not start with '** '", second); + rawLines[0] = second.Text; var fields = new string[FieldCount]; + decimal partLength = 0; for (int n = 1; n <= FieldCount; n++) { - Line line = lines[n + 1]; + Line line = reader.ReadLine("DSTV file ends inside the ST block", length); + rawLines[n] = line.Text; if (line.Text.Length == 0) throw Error($"ST field {n} is empty", line); if (n <= TextFields) { @@ -127,52 +187,41 @@ private static (string Comment, string[] Fields) ReadStBlock(List lines, l { string value = line.Text.Trim(' '); if (!IsDecimal(value)) throw Error($"ST field {n} is not a number", line); - ParseDecimal(value, line); + decimal parsed = ParseDecimal(value, line); + if (n == 9) partLength = parsed; fields[n - 1] = value; } } - for (int i = FieldCount + 2; i < StLines; i++) - if (lines[i].Text.Length != 0) throw Error($"line {i + 1} of the ST block is not empty", lines[i]); - if (!IsCode(lines[StLines].Text)) throw Error("the ST block is not followed by a block code", lines[StLines]); - return (second.Text[3..], fields); + for (int i = FieldCount + 1; i < StLines - 1; i++) + { + Line line = reader.ReadLine("DSTV file ends inside the ST block", length); + rawLines[i] = line.Text; + if (line.Text.Length != 0) throw Error($"line {line.Number} of the ST block is not empty", line); + } + return (second.Text[3..], fields, partLength, rawLines); } - private static Nc1Line[] ReadAk(Line head, Line[] body, List warnings) + private static Nc1Line ValidateAndParseAkLine(Line line, int index) { - if (body.Length == 0) throw Error("AK block has no lines", head); - var result = new Nc1Line[body.Length]; - for (int i = 0; i < body.Length; i++) + string[] tokens = Tokens(line); + if (index == 0) { - string[] tokens = Tokens(body[i]); - if (i == 0) - { - if (tokens.Length != 8 || !IsLabel(tokens[0])) throw Error("the first AK line must hold a label letter and 7 numbers", body[i]); - result[i] = new Nc1Line(tokens[0][0], Numbers(tokens, 1, body[i])); - } - else - { - if (tokens.Length > 0 && IsLabel(tokens[0])) throw Error("only the first AK line may hold a label letter", body[i]); - if (tokens.Length != 7) throw Error("an AK line must hold 7 numbers", body[i]); - result[i] = new Nc1Line(null, Numbers(tokens, 0, body[i])); - } + if (tokens.Length != 8 || !IsLabel(tokens[0])) throw Error("the first AK line must hold a label letter and 7 numbers", line); + return new Nc1Line(tokens[0][0], Numbers(tokens, 1, line)); + } + else + { + if (tokens.Length > 0 && IsLabel(tokens[0])) throw Error("only the first AK line may hold a label letter", line); + if (tokens.Length != 7) throw Error("an AK line must hold 7 numbers", line); + return new Nc1Line(null, Numbers(tokens, 0, line)); } - var first = result[0].Values; - var last = result[^1].Values; - if (first[0].Number != last[0].Number || first[1].Number != last[1].Number) - warnings.Add($"AK block at line {head.Number}: the last point differs from the first, the contour is open"); - return result; } - private static Nc1Line[] ReadBo(Line[] body) + private static Nc1Line ValidateAndParseBoLine(Line line) { - var result = new Nc1Line[body.Length]; - for (int i = 0; i < body.Length; i++) - { - string[] tokens = Tokens(body[i]); - if (tokens.Length != 4 || !IsLabel(tokens[0])) throw Error("a BO line must hold a label letter and 3 numbers", body[i]); - result[i] = new Nc1Line(tokens[0][0], Numbers(tokens, 1, body[i])); - } - return result; + string[] tokens = Tokens(line); + if (tokens.Length != 4 || !IsLabel(tokens[0])) throw Error("a BO line must hold a label letter and 3 numbers", line); + return new Nc1Line(tokens[0][0], Numbers(tokens, 1, line)); } private static string[] Tokens(Line line) => line.Text.Split(' ', StringSplitOptions.RemoveEmptyEntries); diff --git a/src/TdbIO/Internal/TemplateLexer.cs b/src/TdbIO/Internal/TemplateLexer.cs index 8b07d5b..d9bc3e4 100644 --- a/src/TdbIO/Internal/TemplateLexer.cs +++ b/src/TdbIO/Internal/TemplateLexer.cs @@ -19,9 +19,8 @@ internal enum TokenKind /// any other character outside a string is an error. internal static class TemplateLexer { - public static IReadOnlyList Lex(string text) + public static IEnumerable Lex(string text) { - var tokens = new List(); int i = 0; while (i < text.Length) { @@ -34,21 +33,21 @@ public static IReadOnlyList Lex(string text) else if (char.IsAsciiLetter(c) || c == '_') { while (++i < text.Length && (char.IsAsciiLetterOrDigit(text[i]) || text[i] == '_')) { } - tokens.Add(new(TokenKind.Identifier, text[start..i], start)); + yield return new(TokenKind.Identifier, text[start..i], start); } else if (c == '-' || char.IsAsciiDigit(c)) { i = Number(text, i); - tokens.Add(new(TokenKind.Number, text[start..i], start)); + yield return new(TokenKind.Number, text[start..i], start); } else if (c == '"') { (string value, i) = QuotedString(text, i); - tokens.Add(new(TokenKind.String, value, start)); + yield return new(TokenKind.String, value, start); } else if (c is '{' or '}' or '(' or ')' or ';' or '=' or ',') { - tokens.Add(new(TokenKind.Punctuation, c.ToString(), start)); + yield return new(TokenKind.Punctuation, c.ToString(), start); i++; } else @@ -56,8 +55,7 @@ public static IReadOnlyList Lex(string text) throw Error($"character U+{(int)c:X4} is not allowed outside a string", text, i); } } - tokens.Add(new(TokenKind.End, "", text.Length)); - return tokens; + yield return new(TokenKind.End, "", text.Length); } // -?\d+(\.\d+)? ; returns the offset after the number. diff --git a/src/TdbIO/Internal/TemplateParser.cs b/src/TdbIO/Internal/TemplateParser.cs index 5da4a41..5d1c011 100644 --- a/src/TdbIO/Internal/TemplateParser.cs +++ b/src/TdbIO/Internal/TemplateParser.cs @@ -10,36 +10,50 @@ namespace TdbIO.Internal; /// value := STRING | NUMBER | IDENT | "(" value { "," value } ")" /// /// -internal sealed class TemplateParser +internal sealed class TemplateParser : IDisposable { // Guards against hostile input, not properties of the format. public const int MaxLength = 16 * 1024 * 1024; public const int MaxDepth = 64; private readonly string _text; - private readonly IReadOnlyList _tokens; - private int _at; + private readonly IEnumerator _tokens; + private Token _current; + private Token _next; private TemplateParser(string text) { _text = text; - _tokens = TemplateLexer.Lex(text); + _tokens = TemplateLexer.Lex(text).GetEnumerator(); + _current = _tokens.MoveNext() ? _tokens.Current : new Token(TokenKind.End, "", text.Length); + _next = _tokens.MoveNext() ? _tokens.Current : new Token(TokenKind.End, "", text.Length); } + public void Dispose() => _tokens.Dispose(); + public static TemplateBlock Parse(string text) { - var parser = new TemplateParser(text); + using var parser = new TemplateParser(text); var root = parser.Block(0); if (parser.Peek(0).Kind != TokenKind.End) throw parser.Error("only blanks may follow the root block", parser.Peek(0)); return root; } - private Token Peek(int ahead) => _tokens[Math.Min(_at + ahead, _tokens.Count - 1)]; + private Token Peek(int ahead) => ahead switch + { + 0 => _current, + 1 => _next, + _ => throw new ArgumentOutOfRangeException(nameof(ahead)), + }; private Token Next() { - Token token = Peek(0); - if (token.Kind != TokenKind.End) _at++; + Token token = _current; + if (token.Kind != TokenKind.End) + { + _current = _next; + _next = _tokens.MoveNext() ? _tokens.Current : new Token(TokenKind.End, "", _text.Length); + } return token; } @@ -71,7 +85,8 @@ private TemplateBlock Block(int depth) if (token.Kind != TokenKind.Identifier) throw Error("a key, a block or '}' expected", token); if (Is(Peek(1), '=')) { - _at += 2; + Next(); + Next(); properties.Add(new(token.Text, Value(0))); Expect(';'); } diff --git a/tests/TdbIO.Tests/Nc1Tests.cs b/tests/TdbIO.Tests/Nc1Tests.cs index 8a5d147..9bf446f 100644 --- a/tests/TdbIO.Tests/Nc1Tests.cs +++ b/tests/TdbIO.Tests/Nc1Tests.cs @@ -304,4 +304,22 @@ public void FileOverTheSizeLimitIsRejected() var e = Assert.Throws(() => Read(new byte[16 * 1024 * 1024 + 1])); Assert.Contains("16 MiB", e.Message); } + + [Fact] + public void MalformedBoBlockWithManyLinesRejectsEarlyWithoutLargeAllocation() + { + var lines = new List(Synthetic.Nc1Header()); + lines.AddRange(Synthetic.Nc1Rectangle()); + lines.Add("BO"); + for (int i = 0; i < 50_000; i++) lines.Add(""); + lines.Add("EN"); + byte[] data = Synthetic.Nc1(lines); + + long before = GC.GetAllocatedBytesForCurrentThread(); + var e = Assert.Throws(() => Read(data)); + long allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.Contains("a BO line must hold a label letter and 3 numbers", e.Message); + Assert.True(allocated < 500_000, $"Allocated {allocated} bytes, expected < 500 KB"); + } }