Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
235 changes: 142 additions & 93 deletions src/TdbIO/Internal/Nc1Reader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,48 +24,81 @@ public static Nc1Document Read(ReadOnlySpan<byte> data, List<string> warnings)
throw FormatError.At("DSTV file is larger than the 16 MiB limit", MaxFileLength);
if (!FirstLineIsSt(data)) throw FormatError.At("not a DSTV file: line 1 is not ST", 0, 1);

var lines = SplitLines(data);
var (comment, fields) = ReadStBlock(lines, data.Length);
var reader = new LineReader(data);
reader.ReadLine("DSTV file ends inside the ST block", data.Length);

var (comment, fields, length, stRaw) = ReadStBlock(ref reader, data.Length);
var blocks = new List<Nc1Block>
{
new("ST", 1, lines.Skip(1).Take(StLines - 1).Select(l => l.Text).ToArray(), []),
new("ST", 1, stRaw, []),
};

int at = StLines;
while (at < lines.Count)
Line head = reader.ReadLine("DSTV file ends inside the ST block", data.Length);
if (!IsCode(head.Text))
throw Error("the ST block is not followed by a block code", head);

while (head.Text is not null)
{
Line head = lines[at];
int end = at + 1;
while (end < lines.Count && !IsCode(lines[end].Text)) end++;
var body = lines.Skip(at + 1).Take(end - at - 1).ToArray();
var raw = body.Select(l => l.Text).ToArray();
switch (head.Text)
if (head.Text == "ST")
throw Error("an ST block may only start the file", head);

var raw = new List<string>();
var lines = new List<Nc1Line>();
Line nextHead = default;

while (reader.TryReadLine(out Line line))
{
case "ST":
throw Error("an ST block may only start the file", head);
case "EN":
if (body.Length > 0) throw Error("data after the EN block", body[0]);
if (end < lines.Count) throw Error("data after the EN block", lines[end]);
blocks.Add(new("EN", head.Number, raw, []));
break;
case "AK":
blocks.Add(new("AK", head.Number, raw, ReadAk(head, body, warnings)));
break;
case "BO":
blocks.Add(new("BO", head.Number, raw, ReadBo(body)));
break;
default:
// Other codes are not read; their lines are kept as written.
blocks.Add(new(head.Text, head.Number, raw, []));
if (IsCode(line.Text))
{
nextHead = line;
break;
}

switch (head.Text)
{
case "EN":
throw Error("data after the EN block", line);
case "AK":
lines.Add(ValidateAndParseAkLine(line, lines.Count));
break;
case "BO":
lines.Add(ValidateAndParseBoLine(line));
break;
}
raw.Add(line.Text);
}

if (head.Text == "EN")
{
if (nextHead.Text is not null)
throw Error("data after the EN block", nextHead);
blocks.Add(new("EN", head.Number, raw.ToArray(), []));
}
at = end;
else if (head.Text == "AK")
{
if (lines.Count == 0) throw Error("AK block has no lines", head);
var first = lines[0].Values;
var last = lines[^1].Values;
if (first[0].Number != last[0].Number || first[1].Number != last[1].Number)
warnings.Add($"AK block at line {head.Number}: the last point differs from the first, the contour is open");
blocks.Add(new("AK", head.Number, raw.ToArray(), lines.ToArray()));
}
else if (head.Text == "BO")
{
blocks.Add(new("BO", head.Number, raw.ToArray(), lines.ToArray()));
}
else
{
blocks.Add(new(head.Text, head.Number, raw.ToArray(), []));
}

head = nextHead;
}

if (blocks[^1].Code != "EN")
throw FormatError.At("DSTV file is cut: the last block is not EN", data.Length, lines.Count + 1);
throw FormatError.At("DSTV file is cut: the last block is not EN", data.Length, reader.LineNumber + 1);

int? quantity = int.TryParse(fields[5], NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out int q) ? q : null;
decimal length = ParseDecimal(fields[8], lines[2 + 8]);
return new Nc1Document(comment, fields, quantity, length, blocks, warnings);
}

Expand All @@ -75,48 +108,75 @@ private static bool FirstLineIsSt(ReadOnlySpan<byte> data)
return (end < 0 ? data : data[..end]).SequenceEqual("ST"u8);
}

/// <summary>Splits on CR LF only. Every byte is checked: ASCII, no control bytes, no lone CR or LF, and the last line
/// must end with CR LF.</summary>
private static List<Line> SplitLines(ReadOnlySpan<byte> data)
private ref struct LineReader
{
var lines = new List<Line>();
int start = 0;
for (int i = 0; i < data.Length; i++)
private readonly ReadOnlySpan<byte> _data;
private int _position;
private int _lineNumber;

public LineReader(ReadOnlySpan<byte> data)
{
_data = data;
_position = 0;
_lineNumber = 0;
}

public int LineNumber => _lineNumber;

public bool TryReadLine(out Line line)
{
byte b = data[i];
int number = lines.Count + 1;
if (b == '\r')
if (_position >= _data.Length)
{
if (i + 1 >= data.Length || data[i + 1] != '\n')
throw FormatError.At("CR not followed by LF", i, number);
lines.Add(new(Encoding.ASCII.GetString(data[start..i]), start, number));
i++;
start = i + 1;
continue;
line = default;
return false;
}
if (b == '\n') throw FormatError.At("LF without CR", i, number);
if (b >= 0x80) throw FormatError.At($"byte 0x{b:X2} is not ASCII", i, number);
if (b < 0x20 || b == 0x7F) throw FormatError.At($"control byte 0x{b:X2}", i, number);
if (i - start >= MaxLineLength)
throw FormatError.At("line is longer than the 4096 character limit", start, number);

int start = _position;
int number = ++_lineNumber;

for (int i = start; i < _data.Length; i++)
{
byte b = _data[i];
if (b == '\r')
{
if (i + 1 >= _data.Length || _data[i + 1] != '\n')
throw FormatError.At("CR not followed by LF", i, number);
line = new Line(Encoding.ASCII.GetString(_data[start..i]), start, number);
_position = i + 2;
return true;
}
if (b == '\n') throw FormatError.At("LF without CR", i, number);
if (b >= 0x80) throw FormatError.At($"byte 0x{b:X2} is not ASCII", i, number);
if (b < 0x20 || b == 0x7F) throw FormatError.At($"control byte 0x{b:X2}", i, number);
if (i - start >= MaxLineLength)
throw FormatError.At("line is longer than the 4096 character limit", start, number);
}

throw FormatError.At("DSTV file is cut: the last line does not end with CR LF", start, number);
}

public Line ReadLine(string eofMessage, long eofOffset)
{
if (!TryReadLine(out Line line))
throw FormatError.At(eofMessage, eofOffset, _lineNumber + 1);
return line;
}
if (start != data.Length)
throw FormatError.At("DSTV file is cut: the last line does not end with CR LF", start, lines.Count + 1);
return lines;
}

private static (string Comment, string[] Fields) ReadStBlock(List<Line> lines, long length)
private static (string Comment, string[] Fields, decimal Length, string[] RawLines) ReadStBlock(ref LineReader reader, long length)
{
if (lines.Count <= StLines)
throw FormatError.At("DSTV file ends inside the ST block", length, lines.Count + 1);
var rawLines = new string[StLines - 1];

Line second = lines[1];
Line second = reader.ReadLine("DSTV file ends inside the ST block", length);
if (!second.Text.StartsWith("** ", StringComparison.Ordinal)) throw Error("line 2 does not start with '** '", second);
rawLines[0] = second.Text;

var fields = new string[FieldCount];
decimal partLength = 0;
for (int n = 1; n <= FieldCount; n++)
{
Line line = lines[n + 1];
Line line = reader.ReadLine("DSTV file ends inside the ST block", length);
rawLines[n] = line.Text;
if (line.Text.Length == 0) throw Error($"ST field {n} is empty", line);
if (n <= TextFields)
{
Expand All @@ -127,52 +187,41 @@ private static (string Comment, string[] Fields) ReadStBlock(List<Line> lines, l
{
string value = line.Text.Trim(' ');
if (!IsDecimal(value)) throw Error($"ST field {n} is not a number", line);
ParseDecimal(value, line);
decimal parsed = ParseDecimal(value, line);
if (n == 9) partLength = parsed;
fields[n - 1] = value;
}
}
for (int i = FieldCount + 2; i < StLines; i++)
if (lines[i].Text.Length != 0) throw Error($"line {i + 1} of the ST block is not empty", lines[i]);
if (!IsCode(lines[StLines].Text)) throw Error("the ST block is not followed by a block code", lines[StLines]);
return (second.Text[3..], fields);
for (int i = FieldCount + 1; i < StLines - 1; i++)
{
Line line = reader.ReadLine("DSTV file ends inside the ST block", length);
rawLines[i] = line.Text;
if (line.Text.Length != 0) throw Error($"line {line.Number} of the ST block is not empty", line);
}
return (second.Text[3..], fields, partLength, rawLines);
}

private static Nc1Line[] ReadAk(Line head, Line[] body, List<string> warnings)
private static Nc1Line ValidateAndParseAkLine(Line line, int index)
{
if (body.Length == 0) throw Error("AK block has no lines", head);
var result = new Nc1Line[body.Length];
for (int i = 0; i < body.Length; i++)
string[] tokens = Tokens(line);
if (index == 0)
{
string[] tokens = Tokens(body[i]);
if (i == 0)
{
if (tokens.Length != 8 || !IsLabel(tokens[0])) throw Error("the first AK line must hold a label letter and 7 numbers", body[i]);
result[i] = new Nc1Line(tokens[0][0], Numbers(tokens, 1, body[i]));
}
else
{
if (tokens.Length > 0 && IsLabel(tokens[0])) throw Error("only the first AK line may hold a label letter", body[i]);
if (tokens.Length != 7) throw Error("an AK line must hold 7 numbers", body[i]);
result[i] = new Nc1Line(null, Numbers(tokens, 0, body[i]));
}
if (tokens.Length != 8 || !IsLabel(tokens[0])) throw Error("the first AK line must hold a label letter and 7 numbers", line);
return new Nc1Line(tokens[0][0], Numbers(tokens, 1, line));
}
else
{
if (tokens.Length > 0 && IsLabel(tokens[0])) throw Error("only the first AK line may hold a label letter", line);
if (tokens.Length != 7) throw Error("an AK line must hold 7 numbers", line);
return new Nc1Line(null, Numbers(tokens, 0, line));
}
var first = result[0].Values;
var last = result[^1].Values;
if (first[0].Number != last[0].Number || first[1].Number != last[1].Number)
warnings.Add($"AK block at line {head.Number}: the last point differs from the first, the contour is open");
return result;
}

private static Nc1Line[] ReadBo(Line[] body)
private static Nc1Line ValidateAndParseBoLine(Line line)
{
var result = new Nc1Line[body.Length];
for (int i = 0; i < body.Length; i++)
{
string[] tokens = Tokens(body[i]);
if (tokens.Length != 4 || !IsLabel(tokens[0])) throw Error("a BO line must hold a label letter and 3 numbers", body[i]);
result[i] = new Nc1Line(tokens[0][0], Numbers(tokens, 1, body[i]));
}
return result;
string[] tokens = Tokens(line);
if (tokens.Length != 4 || !IsLabel(tokens[0])) throw Error("a BO line must hold a label letter and 3 numbers", line);
return new Nc1Line(tokens[0][0], Numbers(tokens, 1, line));
}

private static string[] Tokens(Line line) => line.Text.Split(' ', StringSplitOptions.RemoveEmptyEntries);
Expand Down
14 changes: 6 additions & 8 deletions src/TdbIO/Internal/TemplateLexer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@ internal enum TokenKind
/// any other character outside a string is an error.</summary>
internal static class TemplateLexer
{
public static IReadOnlyList<Token> Lex(string text)
public static IEnumerable<Token> Lex(string text)
{
var tokens = new List<Token>();
int i = 0;
while (i < text.Length)
{
Expand All @@ -34,30 +33,29 @@ public static IReadOnlyList<Token> Lex(string text)
else if (char.IsAsciiLetter(c) || c == '_')
{
while (++i < text.Length && (char.IsAsciiLetterOrDigit(text[i]) || text[i] == '_')) { }
tokens.Add(new(TokenKind.Identifier, text[start..i], start));
yield return new(TokenKind.Identifier, text[start..i], start);
}
else if (c == '-' || char.IsAsciiDigit(c))
{
i = Number(text, i);
tokens.Add(new(TokenKind.Number, text[start..i], start));
yield return new(TokenKind.Number, text[start..i], start);
}
else if (c == '"')
{
(string value, i) = QuotedString(text, i);
tokens.Add(new(TokenKind.String, value, start));
yield return new(TokenKind.String, value, start);
}
else if (c is '{' or '}' or '(' or ')' or ';' or '=' or ',')
{
tokens.Add(new(TokenKind.Punctuation, c.ToString(), start));
yield return new(TokenKind.Punctuation, c.ToString(), start);
i++;
}
else
{
throw Error($"character U+{(int)c:X4} is not allowed outside a string", text, i);
}
}
tokens.Add(new(TokenKind.End, "", text.Length));
return tokens;
yield return new(TokenKind.End, "", text.Length);
}

// -?\d+(\.\d+)? ; returns the offset after the number.
Expand Down
Loading
Loading