diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index afa1a4b..49837e7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -286,9 +286,11 @@ jobs: working-directory: ./rust run: cargo test --doc --verbose - - name: Run example + - name: Run examples working-directory: ./rust - run: cargo run --example basic_usage + run: | + cargo run --example basic_usage + cargo run --example append_only_log # === BUILD === # Build package - only runs if lint and test pass diff --git a/.gitignore b/.gitignore index af52c44..45f5a72 100644 --- a/.gitignore +++ b/.gitignore @@ -172,3 +172,6 @@ htmlcov/ .dmypy.json dmypy.json .ruff_cache/ + +# Cargo build output of the scratch crates under experiments/ +experiments/**/target/ diff --git a/.gitkeep b/.gitkeep index ac37274..29a8cf9 100644 --- a/.gitkeep +++ b/.gitkeep @@ -1,4 +1,5 @@ # .gitkeep file auto-generated at 2026-05-10T19:22:27.543Z for PR creation at branch issue-35-03946ff48852 for issue https://github.com/link-foundation/lino-objects-codec/issues/35 # Updated: 2026-08-20T05:25:16.696Z # Updated: 2026-08-20T06:10:07.182Z -# Updated: 2026-08-20T07:45:09.136Z \ No newline at end of file +# Updated: 2026-08-20T07:45:09.136Z +# Updated: 2026-08-27T11:41:42.189Z \ No newline at end of file diff --git a/README.md b/README.md index 4ba2830..c299941 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ All implementations share the same design philosophy and provide feature parity. - **C#**: `null`, `bool`, `int`, `long`, `float`, `double`, `string`, `List`, `Dictionary` - Special float/number values: `NaN`, `Infinity`, `-Infinity` - **Readable by Default**: In every language `encode()` writes indented, plain-text Links Notation; the previous single-line base64 form stays available as `encode_compact()` (alias `encode_obfuscated()`) +- **One Record per Line**: `encode_line()` writes the same readable document on one line and `decode_line()` reads it back exactly, so an append-only log stays greppable, tailable and countable by `wc -l` - **Object Identity**: Shared references and circular references are preserved by the compact format via object ids; the readable format is a plain tree and raises a circular-reference error instead - **Full Unicode**: Strings are written as text; only a value that cannot be written as text (one holding control characters) is base64-encoded, and it is marked individually as `(base64 "…")` - **Opt-in Tracing**: Set `LINO_CODEC_DEBUG=1` to trace encoding and decoding, the same way in every language @@ -115,8 +116,37 @@ assert_eq!(decoded, data); ) ``` +For an append-only log, `encode_line()` writes the same document on one line: + +```lino +(o: (name "Alice") (age 30) (active true)) +``` + +```python +from link_notation_objects_codec import encode_line, decode_line + +decode_line(encode_line(data)) == data +``` + +```javascript +import { encodeLine, decodeLine } from "lino-objects-codec"; + +decodeLine({ notation: encodeLine({ obj: data }) }); +``` + +```rust +use lino_objects_codec::{decode_line, encode_line}; + +assert_eq!(decode_line(&encode_line(&data)).unwrap(), data); +``` + +```csharp +var line = Codec.EncodeLine(data); +var record = Codec.DecodeLine(line); +``` + The single-line base64 form is still available as `encode_compact()` (alias -`encode_obfuscated()`) in every language, and `decode()` accepts both forms. +`encode_obfuscated()`) in every language, and `decode()` accepts all three forms. ### C# @@ -395,6 +425,29 @@ object, bare-value lines make an array: - The four languages produce byte-identical output, checked by the shared fixtures in [`fixtures/readable-format/cases.json`](fixtures/readable-format/cases.json) +### Single-line format (`encode_line`) + +The same readable document written on one line, so an append-only log holds one +record per line — appending is one write, compaction cuts at a newline, and +`grep`, `tail -f` and `wc -l` all treat a line as one event: + +```lino +(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1") (port 18878)))) +``` + +- An object is `(o: (key value) …)` and an empty object is `(o:)` +- An array is `(value …)` and an empty array is `()` +- Scalars and strings are written exactly as in the indented form, so a string + keeps its own characters and a number keeps its type +- The `o` marker is what removes the ambiguity a flat layout otherwise has: + without it `((key value))` reads both as a one-pair object and as an array + holding a two-element array. With it a bare `( )` on one line is always an + array, so a *hand-written* `(a 1)` is the two-element array — on one line, + objects say so +- `decode()` reads this form too, so a log reader needs no flag saying which form + a file holds; `decode_line()` is the exact inverse of `encode_line()` and + rejects input spanning more than one line + ### Compact format (`encode_compact`) The previous single-line form, kept for compatibility and for the object graphs diff --git a/csharp/.changeset/20260827_090000_issue_43_single_line_format.md b/csharp/.changeset/20260827_090000_issue_43_single_line_format.md new file mode 100644 index 0000000..72975d9 --- /dev/null +++ b/csharp/.changeset/20260827_090000_issue_43_single_line_format.md @@ -0,0 +1,23 @@ +--- +'Lino.Objects.Codec': minor +--- + +Add `Codec.EncodeLine` and `Codec.DecodeLine` (and the matching `ObjectCodec` +methods): the readable format written on one line, so an append-only log holds +one record per line. Appending is one write, compaction cuts at a newline, and +`grep`, `tail -f` and `wc -l` treat a line as one event. The output is valid +Links Notation, keeps numbers, booleans and `null` bare so types survive the +round trip, and `Decode(EncodeLine(v))` equals `Decode(Encode(v))`. + +`Readable.ObjectMarker` (`o`) tells an object from an array on one line: +`(o: (bytes 2827) (complete true))` is a record, `("a" 1)` is a two-element +array, `(o:)` is the empty object and `()` the empty array. Because the marker +is part of the notation, the empty key round-trips as `(o: ("" 2))`. The +single-line spelling of every shared fixture is pinned in +`fixtures/readable-format/cases.json`, so all four languages write the same +bytes. + +Also fixes `IsCompactNotation`, which used to claim a readable single-line +document such as `(null 1)`; the document `(null)` stays the compact null so +older documents keep decoding. See +[issue #43](https://github.com/link-foundation/lino-objects-codec/issues/43). diff --git a/csharp/README.md b/csharp/README.md index c4aae7a..1c007d2 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -15,6 +15,7 @@ A C# library for working with Links Notation format. This library provides unive - Collections: `List`, `Dictionary` - Special float values: `NaN`, `Infinity`, `-Infinity` - **Readable by Default**: `Codec.Encode()` writes plain, indented text that can be read and reviewed +- **One Record per Line**: `Codec.EncodeLine()` writes the same document on one line and `Codec.DecodeLine()` reads it back exactly, so an append-only log stays greppable, tailable and countable by `wc -l` - **Object Identity**: Shared references and circular references are preserved by the compact format (`Codec.EncodeCompact`) via object ids - **Full Unicode**: Strings are written as text; only a value that cannot be written as text (one holding control characters) is base64-encoded, and it is marked individually as `(base64 "…")` - **Opt-in Tracing**: Set `LINO_CODEC_DEBUG=1` to trace encoding and decoding, the same way in every language @@ -153,6 +154,7 @@ decoded = Codec.Decode(Codec.Encode(complexData)); | --- | --- | | `Codec.Encode(obj)` | Readable, indented Links Notation (the default) | | `Codec.Encode(obj, "\t")` | Same, with a custom indentation string | +| `Codec.EncodeLine(obj)` | The same readable document on one line, for append-only logs | | `Codec.EncodeCompact(obj)` | The previous single-line, base64 form | | `Codec.EncodeObfuscated(obj)` | Alias of `Codec.EncodeCompact` | @@ -217,6 +219,25 @@ bare-value lines make a list: base64-encoded on its own and marked as `(base64 "bGluZTEKbGluZTI=")`; everything around it stays readable +### Single-line format (`Codec.EncodeLine`) + +The same readable document on one line, so an append-only log holds one record +per line -- appending is one write, compaction cuts at a newline, and `grep`, +`tail -f` and `wc -l` all treat a line as one event: + +```lino +(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1") (port 18878)))) +``` + +- A dictionary is `(o: (key value) …)` and an empty dictionary is `(o:)` +- A list is `(value …)` and an empty list is `()` +- Scalars and strings are written exactly as in the indented form +- The `o` marker removes the ambiguity a flat layout otherwise has: a bare `( )` + on one line is always a list, so a *hand-written* `(a 1)` is the two-element + list, not the one-pair dictionary +- `Codec.Decode` reads this form too; `Codec.DecodeLine` is its exact inverse and + rejects input spanning more than one line + ### Compact format (`Codec.EncodeCompact`) The previous single-line form, kept for compatibility and for the object graphs @@ -282,6 +303,34 @@ Decode Links Notation format to a C# object. **Throws:** - `InvalidOperationException` - If the type marker is unknown +#### `Codec.EncodeLine(object? obj)` + +Encode a C# object into the readable format on one line. + +**Parameters:** +- `obj` - The C# object to encode (can be null) + +**Returns:** +- String representation in readable Links Notation format, holding no newline + +```csharp +Codec.EncodeLine(new Dictionary { ["age"] = 30 }); // (o: (age 30)) +``` + +#### `Codec.DecodeLine(string notation)` + +Decode one line of a readable Links Notation log. The exact inverse of +`Codec.EncodeLine`. + +**Parameters:** +- `notation` - One line written by `Codec.EncodeLine` + +**Returns:** +- Reconstructed C# object (or null) + +**Throws:** +- `FormatException` - If the input spans more than one line or is malformed + ### ObjectCodec Class The main codec class that performs encoding and decoding. The static `Codec` class creates a new instance for each operation to ensure thread safety. diff --git a/csharp/examples/BasicUsage.cs b/csharp/examples/BasicUsage.cs index 164e906..4168dbe 100644 --- a/csharp/examples/BasicUsage.cs +++ b/csharp/examples/BasicUsage.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Linq; using Lino.Objects.Codec; Console.WriteLine("=== lino-objects-codec C# Basic Usage Example ===\n"); @@ -146,4 +147,38 @@ Console.WriteLine(); +// An append-only log wants one record per line: appending is one write, a +// compactor can cut the file at any newline, and `grep`, `tail -f` and `wc -l` +// all treat one line as one event. +Console.WriteLine("Append-only log, one record per line:"); + +Dictionary Record(string phase, int bytes, bool complete) => new() +{ + { "phase", phase }, + { "bytes", bytes }, + { "complete", complete } +}; + +var entries = new[] +{ + Record("stream_start", 0, false), + Record("stream_chunk", 1024, false), + Record("stream_end", 2827, true) +}; +var log = string.Concat(entries.Select(entry => Codec.EncodeLine(entry) + "\n")); +Console.Write(log); + +var lines = log.TrimEnd('\n').Split('\n'); +Console.WriteLine($" records: {lines.Length}"); + +// Reading: a line reader hands over one record at a time. +var lastRecord = Codec.DecodeLine(lines[^1]) as Dictionary; +Console.WriteLine($" last record phase: {lastRecord?["phase"]}"); + +// Filtering: the text stays readable, so plain string tools still work. +var finished = lines.Count(line => line.Contains("(complete true)", StringComparison.Ordinal)); +Console.WriteLine($" finished records: {finished}"); + +Console.WriteLine(); + Console.WriteLine("=== Example completed successfully! ==="); diff --git a/csharp/src/Lino.Objects.Codec/ObjectCodec.cs b/csharp/src/Lino.Objects.Codec/ObjectCodec.cs index 24c7aca..56530ca 100644 --- a/csharp/src/Lino.Objects.Codec/ObjectCodec.cs +++ b/csharp/src/Lino.Objects.Codec/ObjectCodec.cs @@ -127,6 +127,26 @@ private void FindObjectsNeedingIds(object? obj, Dictionary? seen = /// String representation in readable Links Notation format public string Encode(object? obj, string indent) => Readable.Encode(obj, indent); + /// + /// Encode a C# object into the readable format on one line. + /// + /// + /// The result holds no newline, so an append-only log keeps one record per + /// line and stays greppable, tailable and countable by wc -l. See + /// for the shape. + /// + /// The C# object to encode + /// String representation in readable Links Notation format, on one line + public string EncodeLine(object? obj) => Readable.EncodeLine(obj); + + /// + /// Decode one line of a readable Links Notation log back into a C# object. + /// + /// The exact inverse of . + /// One line written by + /// Reconstructed C# object + public object? DecodeLine(string notation) => Readable.DecodeLine(notation); + /// /// Encode a C# object to the compact Links Notation format. /// @@ -654,6 +674,18 @@ private Link EncodeValue(object? obj, HashSet? visited = null, i "array", TypeList, "object", TypeDict, }; + /// + /// Markers a compact document writes without a payload. + /// + /// + /// (null) is a compact null, while (null 1) is a readable line + /// holding two values, so the marker alone does not decide the format. + /// + private static readonly HashSet EmptyBodyMarkers = new(StringComparer.Ordinal) + { + TypeNull, "None", + }; + /// /// Whether a document is in the compact format. /// @@ -677,27 +709,41 @@ public static bool IsCompactNotation(string notation) return false; } - var tokens = firstLine[1..] - .Split(new[] { ' ', '\t', '\r', '(', ')' }, StringSplitOptions.RemoveEmptyEntries); - - if (tokens.Length == 0) - { - return false; - } - - var marker = tokens[0]; + var (marker, rest) = SplitToken(firstLine[1..].TrimStart()); // Skip the `obj_N:` definition id, if present. if (marker.EndsWith(':')) { - if (!marker[..^1].StartsWith("obj_", StringComparison.Ordinal) || tokens.Length < 2) + if (!marker.StartsWith("obj_", StringComparison.Ordinal)) { return false; } - marker = tokens[1]; + (marker, rest) = SplitToken(rest.TrimStart()); + } + + if (!CompactTypeMarkers.Contains(marker)) + { + return false; } - return CompactTypeMarkers.Contains(marker); + // A compact `null` carries no payload, so `(null)` is a compact document + // while `(null 1)` is a readable line holding two values. + if (EmptyBodyMarkers.Contains(marker)) + { + return rest.TrimStart().StartsWith(')'); + } + + return true; + } + + /// + /// Split off the first token of a line: everything up to the next whitespace + /// or parenthesis, plus what follows it. + /// + private static (string Token, string Remainder) SplitToken(string input) + { + var end = input.IndexOfAny(new[] { ' ', '\t', '\r', '\n', '(', ')' }); + return end < 0 ? (input, string.Empty) : (input[..end], input[end..]); } } @@ -754,6 +800,20 @@ public static class Codec /// Reconstructed C# object public static object? Decode(string notation) => new ObjectCodec().Decode(notation); + /// + /// Encode an object into the readable format on one line. + /// + /// The C# object to encode + /// String representation in readable Links Notation format, on one line + public static string EncodeLine(object? obj) => new ObjectCodec().EncodeLine(obj); + + /// + /// Decode one line of a readable Links Notation log. + /// + /// One line written by + /// Reconstructed C# object + public static object? DecodeLine(string notation) => new ObjectCodec().DecodeLine(notation); + /// /// Decode the compact Links Notation format to a C# object. /// diff --git a/csharp/src/Lino.Objects.Codec/Readable.cs b/csharp/src/Lino.Objects.Codec/Readable.cs index 36f083d..01958c5 100644 --- a/csharp/src/Lino.Objects.Codec/Readable.cs +++ b/csharp/src/Lino.Objects.Codec/Readable.cs @@ -75,6 +75,30 @@ public CircularReferenceException(string message, Exception innerException) /// CRLF normalisation would corrupt) are marked individually as /// (base64 "…") instead of encoding the whole document. /// +/// +/// writes the same document on one line, so one record +/// is one line and an append-only log stays greppable, tailable and countable by +/// wc -l. Rows can no longer be told apart by line breaks there, so an +/// object names itself with the o link id the notation already has, and +/// its pairs are written as their own links: +/// +/// +/// (o: (type "RouterState") (server (o: (host "127.0.0.1") (port 18878))) (models ("claude-haiku" "claude-opus"))) +/// +/// +/// An object is (o: (key value) …) and an empty one is (o:); an +/// array is (value …) and an empty one is (); scalars are written +/// exactly as in the indented form. +/// +/// +/// The marker is what answers the ambiguity a flat layout otherwise has: without +/// it ((key value)) reads both as the one-pair object and as the array +/// holding the two-element array, and an empty key makes it worse. With it, a +/// bare ( ) is always an array and a marked one is always an object, so +/// every value — empty key included — survives the round trip. Consequently a +/// hand-written one-line link such as (a 1) is the two-element +/// array, not the one-pair object: on one line, objects say so. +/// /// public static class Readable { @@ -84,6 +108,9 @@ public static class Readable /// Marker used for values that cannot be represented as plain text. public const string Base64Marker = "base64"; + /// Link id naming an object in the single-line form, written as (o: …). + public const string ObjectMarker = "o"; + /// Characters that cannot appear in a bare (unquoted) reference. private static readonly char[] QuoteChars = { '"', '\'', '`' }; @@ -112,6 +139,44 @@ public static string Encode(object? value, string indent) /// The readable Links Notation document public static string Encode(object? value) => Encode(value, DefaultIndent); + /// + /// Encode a value into the readable, single-line Links Notation form. + /// + /// + /// The result never contains a newline, so one value is one line of an + /// append-only log. See the class documentation for the shape. + /// + /// The value to encode + /// The readable Links Notation document, on one line + /// If the value refers back to itself + public static string EncodeLine(object? value) + { + var output = new StringBuilder(); + WriteLineValue(value, output, new HashSet(ReferenceEqualityComparer.Instance)); + return output.ToString(); + } + + /// + /// Decode the readable, single-line Links Notation form back into a value. + /// + /// + /// This is the exact inverse of . Input spanning more + /// than one line is rejected: a line-based reader hands over one record at a + /// time, and silently accepting several would merge two records into one value. + /// + /// One line of a readable Links Notation log + /// The reconstructed value + /// If the document is not well formed or holds a line break + public static object? DecodeLine(string text) + { + var line = text.Trim('\n', '\r'); + if (line.Contains('\n') || line.Contains('\r')) + { + throw new FormatException("a single-line document cannot contain a line break"); + } + return Decode(line); + } + /// /// Decode the readable, indented Links Notation form back into a value. /// @@ -136,7 +201,7 @@ public static string Encode(object? value, string indent) return NodeToValue(rows[0][0]); } - return RowsToValue(rows, true); + return RowsToValue(rows, true, false); } // === Encoding === @@ -200,6 +265,58 @@ private static void WriteValue(object? value, string indent, int level, StringBu output.Append(FormatScalar(value)); } + /// + /// Write a value on one line. Objects name themselves with the o link + /// id and write each pair as its own link, so nothing depends on where lines + /// break. + /// + private static void WriteLineValue(object? value, StringBuilder output, HashSet path) + { + if (value is IDictionary dict) + { + EnterPath(dict, path); + if (dict.Count == 0) + { + // `()` is the empty array, so the empty object keeps its marker. + output.Append('(').Append(ObjectMarker).Append(":)"); + } + else + { + output.Append('(').Append(ObjectMarker).Append(':'); + foreach (var pair in dict) + { + output.Append(" (").Append(FormatKey(pair.Key)).Append(' '); + WriteLineValue(pair.Value, output, path); + output.Append(')'); + } + output.Append(')'); + } + path.Remove(dict); + return; + } + + if (value is System.Collections.IEnumerable items and not string) + { + EnterPath(items, path); + output.Append('('); + var first = true; + foreach (var item in items) + { + if (!first) + { + output.Append(' '); + } + first = false; + WriteLineValue(item, output, path); + } + output.Append(')'); + path.Remove(items); + return; + } + + output.Append(FormatScalar(value)); + } + /// /// Mark a container as being written, so a reference back to it is caught. /// @@ -349,6 +466,9 @@ private sealed class Node public bool Quoted { get; init; } public List> Rows { get; init; } = new(); public bool Multiline { get; init; } + + /// Whether the link named itself an object with the o: marker. + public bool IsObject { get; init; } } /// @@ -511,14 +631,35 @@ private Node ParseNode() if (token.Kind == TokenKind.Open) { Pos++; + var isObject = TakeObjectMarker(); var multiline = LinkIsMultiline(); var rows = ParseRows(false); - return new Node { IsRef = false, Rows = rows, Multiline = multiline }; + return new Node { IsRef = false, Rows = rows, Multiline = multiline, IsObject = isObject }; } throw new FormatException("unexpected token in readable notation"); } + /// + /// Consume the o: marker if the link that just opened carries one, + /// which is how the single-line form says "this link is an object, not an + /// array". + /// + private bool TakeObjectMarker() + { + if (Pos >= _tokens.Count) + { + return false; + } + var token = _tokens[Pos]; + if (token.Kind != TokenKind.Ref || token.Quoted || token.Value != ObjectMarker + ":") + { + return false; + } + Pos++; + return true; + } + /// /// Whether the link that just opened spans more than one line, which is /// what tells an empty object ((\n)) from an empty array (()). @@ -541,10 +682,17 @@ private bool LinkIsMultiline() } private static object? NodeToValue(Node node) => - node.IsRef ? RefToValue(node.Value, node.Quoted) : RowsToValue(node.Rows, node.Multiline); + node.IsRef + ? RefToValue(node.Value, node.Quoted) + : RowsToValue(node.Rows, node.Multiline, node.IsObject); - private static object? RowsToValue(List> rows, bool multiline) + private static object? RowsToValue(List> rows, bool multiline, bool objectMarker) { + if (objectMarker) + { + return MarkedObjectToValue(rows); + } + if (rows.Count == 0) { return multiline ? new Dictionary() : new List(); @@ -556,6 +704,22 @@ private bool LinkIsMultiline() return marked; } + // Written on one line, a link is a list of values: an object on one line + // says so with the `o:` marker, which is what keeps `(key value)` + // unambiguous. + if (!multiline) + { + var line = new List(); + foreach (var row in rows) + { + foreach (var node in row) + { + line.Add(NodeToValue(node)); + } + } + return line; + } + // `key value` on every line makes an object; anything else is a list of values. var isObject = rows.All(row => row.Count == 2 && row[0].IsRef); @@ -580,6 +744,42 @@ private bool LinkIsMultiline() return items; } + /// + /// Build the object a (o: (key value) …) link describes. Every value in + /// it is a pair, so anything else is a malformed document rather than a + /// silent array. + /// + private static object? MarkedObjectToValue(List> rows) + { + var result = new Dictionary(); + + foreach (var node in rows.SelectMany(row => row)) + { + if (node.IsRef || node.IsObject) + { + throw new FormatException( + $"an object marked '{ObjectMarker}:' holds (key value) pairs, " + + "found a value that is not a pair"); + } + if (node.Rows.Count != 1) + { + throw new FormatException( + $"an object marked '{ObjectMarker}:' holds (key value) pairs, " + + $"found a link of {node.Rows.Count.ToString(CultureInfo.InvariantCulture)} lines"); + } + var pair = node.Rows[0]; + if (pair.Count != 2 || !pair[0].IsRef) + { + throw new FormatException( + $"an object marked '{ObjectMarker}:' holds (key value) pairs, " + + $"found a link of {pair.Count.ToString(CultureInfo.InvariantCulture)} values"); + } + result[pair[0].Value] = NodeToValue(pair[1]); + } + + return result; + } + /// /// Recognise (base64 "…"), the individual marker for values that could /// not be written as text. A quoted base64 key is an ordinary object diff --git a/csharp/tests/Lino.Objects.Codec.Tests/ReadableConformanceTests.cs b/csharp/tests/Lino.Objects.Codec.Tests/ReadableConformanceTests.cs index d53ac6f..b278b13 100644 --- a/csharp/tests/Lino.Objects.Codec.Tests/ReadableConformanceTests.cs +++ b/csharp/tests/Lino.Objects.Codec.Tests/ReadableConformanceTests.cs @@ -1,10 +1,11 @@ -// Cross-language conformance tests for the readable, indented format. +// Cross-language conformance tests for the readable format, indented and on a +// single line. // // The fixtures in fixtures/readable-format/cases.json are shared by the // JavaScript, Python, Rust and C# suites. Each case is written by hand from the // format specification, so the four implementations check each other instead of // agreeing on a shared mistake: every language must encode `value` to exactly -// `text` and decode `text` back to exactly `value`. +// `text` and to exactly `line`, and decode both back to exactly `value`. using System.Globalization; using System.Text.Json; @@ -178,4 +179,58 @@ public void DecodesEachSharedTextBackToTheCaseValue(string name, JsonElement @ca var decoded = Codec.Decode(@case.GetProperty("text").GetString()!); Assert.True(Same(expected, decoded), $"case {name} decoded to a different value"); } + + [Theory] + [MemberData(nameof(AllCases))] + public void EncodesEachCaseToTheSharedLine(string name, JsonElement @case) + { + _ = name; + if (IsSkipped(@case)) + { + return; + } + var encoded = Codec.EncodeLine(Build(@case.GetProperty("value"))); + Assert.Equal(@case.GetProperty("line").GetString(), encoded); + } + + [Theory] + [MemberData(nameof(AllCases))] + public void DecodesEachSharedLineBackToTheCaseValue(string name, JsonElement @case) + { + if (IsSkipped(@case)) + { + return; + } + var expected = Build(@case.GetProperty("value")); + var decoded = Codec.DecodeLine(@case.GetProperty("line").GetString()!); + Assert.True(Same(expected, decoded), $"case {name} decoded to a different value"); + } + + /// A log record is one line, so no case may spread over two of them. + [Theory] + [MemberData(nameof(AllCases))] + public void NoSharedLineContainsALineBreak(string name, JsonElement @case) + { + var line = @case.GetProperty("line").GetString()!; + Assert.True( + !line.Contains('\n') && !line.Contains('\r'), + $"case {name} has a line break in its single-line form"); + } + + /// + /// reads both forms, so a log reader needs no flag + /// saying which one it holds. + /// + [Theory] + [MemberData(nameof(AllCases))] + public void ThePlainDecoderReadsEachSharedLine(string name, JsonElement @case) + { + if (IsSkipped(@case)) + { + return; + } + var expected = Build(@case.GetProperty("value")); + var decoded = Codec.Decode(@case.GetProperty("line").GetString()!); + Assert.True(Same(expected, decoded), $"case {name} decoded to a different value"); + } } diff --git a/csharp/tests/Lino.Objects.Codec.Tests/SingleLineFormatTests.cs b/csharp/tests/Lino.Objects.Codec.Tests/SingleLineFormatTests.cs new file mode 100644 index 0000000..5bc603a --- /dev/null +++ b/csharp/tests/Lino.Objects.Codec.Tests/SingleLineFormatTests.cs @@ -0,0 +1,232 @@ +// Tests for the readable, single-line format produced by EncodeLine (issue #43). +// +// An append-only log wants one record per line: appending is one write, +// compaction cuts at a newline, and `grep`, `tail -f` and `wc -l` all treat a +// line as an event. Encode spreads a record over many lines and EncodeCompact +// hides it in base64, so neither serves that reader. + +using Xunit; +using Link.Foundation.Links.Notation; +using Lino.Objects.Codec; + +namespace Lino.Objects.Codec.Tests; + +/// +/// Checks the single-line form: one record, one line, read back exactly. +/// +public class SingleLineFormatTests +{ + /// A record of the shape an append-only log actually holds. + private static Dictionary LogRecord() => new() + { + ["bytes"] = 2827, + ["complete"] = true, + ["server"] = new Dictionary + { + ["host"] = "127.0.0.1", + ["port"] = 18878, + }, + ["models"] = new List { "claude-haiku", "claude-opus" }, + }; + + /// The dialect a downstream project invented for the same need. + private const string HandRolledDialect = "((:\"bytes\" 2827) (:\"complete\" true))"; + + [Fact] + public void ARecordIsWrittenOnOneLine() + { + var line = Codec.EncodeLine(LogRecord()); + Assert.DoesNotContain('\n', line); + Assert.DoesNotContain('\r', line); + Assert.Equal( + "(o: (bytes 2827) (complete true) (server (o: (host \"127.0.0.1\") (port 18878))) " + + "(models (\"claude-haiku\" \"claude-opus\")))", + line); + } + + [Fact] + public void ALineIsValidLinksNotation() + { + var line = Codec.EncodeLine(LogRecord()); + var links = new Parser().Parse(line); + Assert.NotNull(links); + Assert.NotEmpty(links); + } + + /// + /// The hand-rolled dialect is what this format replaces: it does not read + /// back as the record it was written from. + /// + [Fact] + public void TheHandRolledDialectIsNotReadAsARecord() + { + var decoded = Codec.Decode(HandRolledDialect); + Assert.False( + decoded is IDictionary dict + && dict.ContainsKey("bytes") + && dict.ContainsKey("complete"), + $"the hand-rolled dialect unexpectedly read back as a record: {decoded}"); + } + + [Fact] + public void BothFormsOfTheSameValueDecodeAlike() + { + var values = new object?[] + { + LogRecord(), + new List(), + new Dictionary(), + new List { new Dictionary(), new List() }, + new Dictionary { ["empty"] = new List() }, + 42, + null, + }; + + foreach (var value in values) + { + var fromLine = Codec.Decode(Codec.EncodeLine(value)); + var fromText = Codec.Decode(Codec.Encode(value)); + Assert.True(Equivalent(fromLine, fromText), $"the two forms disagree about {value}"); + Assert.True(Equivalent(Codec.DecodeLine(Codec.EncodeLine(value)), value)); + } + } + + [Fact] + public void AStringKeepsItsOwnCharactersOnOneLine() + { + var value = new Dictionary { ["text"] = "quote \" backslash \\ ünïcödé" }; + var line = Codec.EncodeLine(value); + Assert.Equal("(o: (text 'quote \" backslash \\ ünïcödé'))", line); + Assert.True(Equivalent(Codec.DecodeLine(line), value)); + } + + /// + /// A newline inside a string would end the record, so such a string is the + /// one thing written encoded -- individually, so the rest stays readable. + /// + [Fact] + public void AStringHoldingANewlineStillFitsOnOneLine() + { + var value = new Dictionary + { + ["readable"] = "still visible", + ["multiline"] = "line1\nline2", + }; + var line = Codec.EncodeLine(value); + Assert.Equal( + "(o: (readable \"still visible\") (multiline (base64 \"bGluZTEKbGluZTI=\")))", + line); + Assert.DoesNotContain('\n', line); + Assert.True(Equivalent(Codec.DecodeLine(line), value)); + } + + /// + /// The one ambiguity a flat layout has: is (a 1) a one-pair object or + /// a two-element array? On one line an object says so with the o: + /// marker, so both values keep their own spelling. + /// + [Fact] + public void AOnePairObjectIsNotATwoElementArray() + { + var @object = new Dictionary { ["a"] = 1 }; + var array = new List { "a", 1 }; + + Assert.Equal("(o: (a 1))", Codec.EncodeLine(@object)); + Assert.Equal("(\"a\" 1)", Codec.EncodeLine(array)); + Assert.True(Equivalent(Codec.DecodeLine("(o: (a 1))"), @object)); + Assert.True(Equivalent(Codec.DecodeLine("(\"a\" 1)"), array)); + } + + /// + /// Because the marker answers it, the empty key round-trips instead of being + /// rejected: ("" 2) is a pair like any other inside a marked object. + /// + [Fact] + public void TheEmptyKeySurvivesTheRoundTrip() + { + var value = new Dictionary { [""] = 2 }; + Assert.Equal("(o: (\"\" 2))", Codec.EncodeLine(value)); + Assert.True(Equivalent(Codec.DecodeLine(Codec.EncodeLine(value)), value)); + } + + [Fact] + public void AMarkedObjectHoldingSomethingThatIsNotAPairIsRejected() + { + var error = Assert.Throws(() => Codec.DecodeLine("(o: 1 2)")); + Assert.Contains("pairs", error.Message, StringComparison.Ordinal); + } + + /// + /// Reading a log means handing over one record at a time, so a decoder that + /// silently accepted two lines would merge two records into one value. + /// + [Fact] + public void SeveralLinesAreNotOneRecord() + { + Assert.Throws(() => Codec.DecodeLine("(o: (a 1))\n(o: (b 2))")); + } + + /// + /// A trailing newline is what a line reader may keep, so it is trimmed rather + /// than refused. + /// + [Fact] + public void ATrailingNewlineIsNotASecondRecord() + { + var decoded = Codec.DecodeLine("(o: (a 1))\n"); + Assert.True(Equivalent(decoded, new Dictionary { ["a"] = 1 })); + } + + /// + /// A line whose first value is null is a readable line, not the compact null: + /// Decode must not route it to the base64 reader. + /// + [Fact] + public void ALineStartingWithNullIsStillReadAsALine() + { + Assert.True(Equivalent(Codec.Decode("(null 1)"), new List { null, 1 })); + Assert.True(Equivalent( + Codec.Decode("(o: (a null))"), + new Dictionary { ["a"] = null })); + // The one document both forms claim: `(null)` is the compact null, and + // stays read that way, so documents written before this format keep + // decoding. + Assert.Null(Codec.Decode("(null)")); + } + + /// + /// The JavaScript sibling trimmed the framing newlines with a regular + /// expression that backtracked once per newline (CodeQL js/polynomial-redos). + /// Every language strips them with a linear scan instead, and still refuses + /// input holding more than one line. + /// + [Fact] + public void ALongRunOfLineBreaksIsRejectedWithoutASlowdown() + { + var notation = Codec.EncodeLine(LogRecord()) + new string('\n', 200_000) + "x"; + var started = System.Diagnostics.Stopwatch.StartNew(); + Assert.Throws(() => Codec.DecodeLine(notation)); + Assert.True(started.Elapsed < TimeSpan.FromSeconds(2), $"took {started.Elapsed}"); + } + + /// Structural comparison, since dictionaries and lists compare by reference. + private static bool Equivalent(object? left, object? right) + { + switch (left, right) + { + case (null, null): + return true; + case (IDictionary a, IDictionary b): + return a.Count == b.Count + && a.Keys.SequenceEqual(b.Keys) + && a.All(entry => Equivalent(entry.Value, b[entry.Key])); + case (System.Collections.IEnumerable a and not string, System.Collections.IEnumerable b and not string): + var left_items = a.Cast().ToList(); + var right_items = b.Cast().ToList(); + return left_items.Count == right_items.Count + && left_items.Zip(right_items).All(pair => Equivalent(pair.First, pair.Second)); + default: + return Equals(left, right); + } + } +} diff --git a/experiments/issue-43/build_line_fixtures.py b/experiments/issue-43/build_line_fixtures.py new file mode 100644 index 0000000..75dea68 --- /dev/null +++ b/experiments/issue-43/build_line_fixtures.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Fill in the `line` field of the shared readable-format fixtures. + +Written directly from the single-line specification in `rust/src/readable.rs`, +independently of any implementation, so that the four language suites are +checked against the format rather than against one of them. + +Usage: python3 experiments/issue-43/build_line_fixtures.py [--write] +""" +import base64 +import json +import pathlib +import sys + +FIXTURES = pathlib.Path(__file__).resolve().parents[2] / "fixtures/readable-format/cases.json" +BASE64_MARKER = "base64" +OBJECT_MARKER = "o" +KEY_NEEDS_QUOTES = set(" \t\n\r()'\":`") + + +def needs_encoding(text): + return any(ord(c) <= 0x1F or 0x7F <= ord(c) <= 0x9F for c in text) + + +def quote(text): + if '"' not in text: + return f'"{text}"' + if "'" not in text: + return f"'{text}'" + return '"' + text.replace('"', '""') + '"' + + +def format_string(text): + if needs_encoding(text): + payload = base64.b64encode(text.encode("utf-8")).decode("ascii") + return f"({BASE64_MARKER} {quote(payload)})" + return quote(text) + + +def format_key(key): + plain = ( + key != "" + and key != BASE64_MARKER + and not needs_encoding(key) + and not any(c in KEY_NEEDS_QUOTES or c.isspace() for c in key) + ) + return key if plain else format_string(key) + + +def format_float(value): + if isinstance(value, str): + return value # "NaN", "Infinity", "-Infinity" + text = repr(float(value)) + return text + + +def line_of(tagged): + (tag, payload), = tagged.items() + if tag == "null": + return "null" + if tag == "bool": + return "true" if payload else "false" + if tag == "int": + return str(payload) + if tag == "float": + return format_float(payload) + if tag == "str": + return format_string(payload) + if tag == "array": + return "(" + " ".join(line_of(item) for item in payload) + ")" + if tag == "object": + if not payload: + return f"({OBJECT_MARKER}:)" + pairs = " ".join(f"({format_key(k)} {line_of(v)})" for k, v in payload) + return f"({OBJECT_MARKER}: {pairs})" + raise SystemExit(f"unknown tag {tag}") + + +def main(): + document = json.loads(FIXTURES.read_text()) + for case in document["cases"]: + case["line"] = line_of(case["value"]) + print(f"{case['name']}: {case['line']}") + + if "--write" in sys.argv: + FIXTURES.write_text(json.dumps(document, indent=2, ensure_ascii=False) + "\n") + print(f"\nwrote {FIXTURES}") + + +if __name__ == "__main__": + main() diff --git a/experiments/issue-43/line-format-probe/Cargo.lock b/experiments/issue-43/line-format-probe/Cargo.lock new file mode 100644 index 0000000..6799abd --- /dev/null +++ b/experiments/issue-43/line-format-probe/Cargo.lock @@ -0,0 +1,81 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "line-format-probe" +version = "0.1.0" +dependencies = [ + "links-notation", +] + +[[package]] +name = "links-notation" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6e5f36d99612ea82da43dbd5efb37b0b4e9c4b8197228e073b60ef5a0e78f2" +dependencies = [ + "links-notation-macro", + "nom", +] + +[[package]] +name = "links-notation-macro" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30ea96250240a92d69d45579dbd199a713e7a79acfa53033d24628dad23cff5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" diff --git a/experiments/issue-43/line-format-probe/Cargo.toml b/experiments/issue-43/line-format-probe/Cargo.toml new file mode 100644 index 0000000..e5a541f --- /dev/null +++ b/experiments/issue-43/line-format-probe/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "line-format-probe" +version = "0.1.0" +edition = "2021" + +# Probe for issue #43: which single-line spellings does the notation's own +# parser accept, and what shape do they parse into? +[dependencies] +links-notation = "0.14.0" diff --git a/experiments/issue-43/line-format-probe/src/main.rs b/experiments/issue-43/line-format-probe/src/main.rs new file mode 100644 index 0000000..4bed160 --- /dev/null +++ b/experiments/issue-43/line-format-probe/src/main.rs @@ -0,0 +1,33 @@ +use links_notation::parse_lino; + +fn main() { + let candidates = [ + // What the downstream project invented (expected: rejected). + "((:\"bytes\" 2827) (:\"complete\" true))", + // Bare groups: arrays. + "()", + "(1 2)", + "(\"key\" \"value\")", + "((1 2) (3))", + // Object marker written as a link id, which the notation already has. + "(o:)", + "(o: (a 1))", + "(o: (a 1) (b 2))", + "(o: (\"two words\" 1) (\"\" 2) (\"base64\" 3))", + "(o: (a (o: (b (o: (c (1)))))))", + "((o: (id \"1\")) (o: (id \"2\")))", + "(o: (multiline (base64 \"bGluZTEKbGluZTI=\")))", + "(o: (s 'he said \"hello\"') (t \"both \"\"kinds\"\" of 'quotes'\"))", + // Spellings we do not emit, recorded for completeness. + "(o : (a 1))", + "(a: 1 2)", + "(a:)", + ]; + + for candidate in candidates { + match parse_lino(candidate) { + Ok(parsed) => println!("OK {candidate}\n {parsed:?}"), + Err(error) => println!("ERR {candidate}\n {error}"), + } + } +} diff --git a/fixtures/readable-format/cases.json b/fixtures/readable-format/cases.json index e2b3f1e..c6465fe 100644 --- a/fixtures/readable-format/cases.json +++ b/fixtures/readable-format/cases.json @@ -1,5 +1,5 @@ { - "description": "Shared conformance fixtures for the readable, indented Links Notation format. Every implementation must encode `value` to exactly `text` and decode `text` back to exactly `value`, so the four languages produce byte-identical documents.", + "description": "Shared conformance fixtures for the readable Links Notation format. Every implementation must encode `value` to exactly `text` in the indented form and to exactly `line` in the single-line form, and decode both back to exactly `value`, so the four languages produce byte-identical documents.", "valueEncoding": "A value is a single-key object naming its type: {\"null\": true}, {\"bool\": …}, {\"int\": …}, {\"float\": … | \"NaN\" | \"Infinity\" | \"-Infinity\"}, {\"str\": …}, {\"array\": [value, …]} or {\"object\": [[key, value], …]}. Object pairs are a list, not a map, because key order is part of the document.", "skip": "`skip` maps a language id (js, python, rust, csharp) to the reason its value model cannot represent the case. A suite skips exactly the cases naming it.", "cases": [ @@ -8,49 +8,56 @@ "value": { "null": true }, - "text": "null" + "text": "null", + "line": "null" }, { "name": "bool_true", "value": { "bool": true }, - "text": "true" + "text": "true", + "line": "true" }, { "name": "bool_false", "value": { "bool": false }, - "text": "false" + "text": "false", + "line": "false" }, { "name": "int_positive", "value": { "int": 42 }, - "text": "42" + "text": "42", + "line": "42" }, { "name": "int_negative", "value": { "int": -7 }, - "text": "-7" + "text": "-7", + "line": "-7" }, { "name": "int_zero", "value": { "int": 0 }, - "text": "0" + "text": "0", + "line": "0" }, { "name": "float_fraction", "value": { "float": 3.5 }, - "text": "3.5" + "text": "3.5", + "line": "3.5" }, { "name": "float_whole", @@ -60,133 +67,152 @@ "text": "2.0", "skip": { "js": "JavaScript has one number type, so 2.0 and 2 are the same value and the trailing '.0' cannot be recovered when encoding" - } + }, + "line": "2.0" }, { "name": "float_negative", "value": { "float": -0.5 }, - "text": "-0.5" + "text": "-0.5", + "line": "-0.5" }, { "name": "float_nan", "value": { "float": "NaN" }, - "text": "NaN" + "text": "NaN", + "line": "NaN" }, { "name": "float_infinity", "value": { "float": "Infinity" }, - "text": "Infinity" + "text": "Infinity", + "line": "Infinity" }, { "name": "float_negative_infinity", "value": { "float": "-Infinity" }, - "text": "-Infinity" + "text": "-Infinity", + "line": "-Infinity" }, { "name": "string_plain", "value": { "str": "root" }, - "text": "\"root\"" + "text": "\"root\"", + "line": "\"root\"" }, { "name": "string_empty", "value": { "str": "" }, - "text": "\"\"" + "text": "\"\"", + "line": "\"\"" }, { "name": "string_with_spaces", "value": { "str": "with spaces" }, - "text": "\"with spaces\"" + "text": "\"with spaces\"", + "line": "\"with spaces\"" }, { "name": "string_apostrophe", "value": { "str": "it's" }, - "text": "\"it's\"" + "text": "\"it's\"", + "line": "\"it's\"" }, { "name": "string_double_quote", "value": { "str": "he said \"hello\"" }, - "text": "'he said \"hello\"'" + "text": "'he said \"hello\"'", + "line": "'he said \"hello\"'" }, { "name": "string_both_quote_kinds", "value": { "str": "both \"kinds\" of 'quotes'" }, - "text": "\"both \"\"kinds\"\" of 'quotes'\"" + "text": "\"both \"\"kinds\"\" of 'quotes'\"", + "line": "\"both \"\"kinds\"\" of 'quotes'\"" }, { "name": "string_unicode", "value": { "str": "unicode: 你好世界 🌍" }, - "text": "\"unicode: 你好世界 🌍\"" + "text": "\"unicode: 你好世界 🌍\"", + "line": "\"unicode: 你好世界 🌍\"" }, { "name": "string_parens_and_colon", "value": { "str": "parens (and) colons: yes" }, - "text": "\"parens (and) colons: yes\"" + "text": "\"parens (and) colons: yes\"", + "line": "\"parens (and) colons: yes\"" }, { "name": "string_numeric", "value": { "str": "18878" }, - "text": "\"18878\"" + "text": "\"18878\"", + "line": "\"18878\"" }, { "name": "string_boolean", "value": { "str": "true" }, - "text": "\"true\"" + "text": "\"true\"", + "line": "\"true\"" }, { "name": "string_with_newline", "value": { "str": "line1\nline2" }, - "text": "(base64 \"bGluZTEKbGluZTI=\")" + "text": "(base64 \"bGluZTEKbGluZTI=\")", + "line": "(base64 \"bGluZTEKbGluZTI=\")" }, { "name": "string_with_tab", "value": { "str": "a\tb" }, - "text": "(base64 \"YQli\")" + "text": "(base64 \"YQli\")", + "line": "(base64 \"YQli\")" }, { "name": "empty_array", "value": { "array": [] }, - "text": "()" + "text": "()", + "line": "()" }, { "name": "empty_object", "value": { "object": [] }, - "text": "(\n)" + "text": "(\n)", + "line": "(o:)" }, { "name": "array_of_scalars", @@ -206,7 +232,8 @@ } ] }, - "text": "(\n 1\n \"two\"\n true\n null\n)" + "text": "(\n 1\n \"two\"\n true\n null\n)", + "line": "(1 \"two\" true null)" }, { "name": "object_of_scalars", @@ -226,7 +253,8 @@ ] ] }, - "text": "(\n name \"Alice\"\n age 30\n)" + "text": "(\n name \"Alice\"\n age 30\n)", + "line": "(o: (name \"Alice\") (age 30))" }, { "name": "nested_empty_containers", @@ -246,7 +274,8 @@ ] ] }, - "text": "(\n empty_array ()\n empty_object (\n )\n)" + "text": "(\n empty_array ()\n empty_object (\n )\n)", + "line": "(o: (empty_array ()) (empty_object (o:)))" }, { "name": "array_of_objects_keeps_record_boundaries", @@ -286,7 +315,8 @@ } ] }, - "text": "(\n (\n id \"1\"\n label \"one\"\n )\n (\n id \"2\"\n label \"two\"\n )\n)" + "text": "(\n (\n id \"1\"\n label \"one\"\n )\n (\n id \"2\"\n label \"two\"\n )\n)", + "line": "((o: (id \"1\") (label \"one\")) (o: (id \"2\") (label \"two\")))" }, { "name": "array_of_arrays", @@ -311,7 +341,95 @@ } ] }, - "text": "(\n (\n 1\n 2\n )\n (\n 3\n )\n)" + "text": "(\n (\n 1\n 2\n )\n (\n 3\n )\n)", + "line": "((1 2) (3))" + }, + { + "name": "array_of_pairs_is_not_an_object", + "value": { + "array": [ + { + "array": [ + { + "str": "a" + }, + { + "int": 1 + } + ] + }, + { + "array": [ + { + "str": "b" + }, + { + "int": 2 + } + ] + } + ] + }, + "text": "(\n (\n \"a\"\n 1\n )\n (\n \"b\"\n 2\n )\n)", + "line": "((\"a\" 1) (\"b\" 2))" + }, + { + "name": "array_holding_one_object", + "value": { + "array": [ + { + "object": [ + [ + "a", + { + "int": 1 + } + ] + ] + } + ] + }, + "text": "(\n (\n a 1\n )\n)", + "line": "((o: (a 1)))" + }, + { + "name": "array_holding_an_empty_object", + "value": { + "array": [ + { + "object": [] + } + ] + }, + "text": "(\n (\n )\n)", + "line": "((o:))" + }, + { + "name": "log_record_of_one_line", + "value": { + "object": [ + [ + "bytes", + { + "int": 2827 + } + ], + [ + "complete", + { + "bool": true + } + ], + [ + "phase", + { + "str": "stream_end" + } + ] + ] + }, + "text": "(\n bytes 2827\n complete true\n phase \"stream_end\"\n)", + "line": "(o: (bytes 2827) (complete true) (phase \"stream_end\"))" }, { "name": "single_pair_object_is_not_a_two_element_array", @@ -325,7 +443,8 @@ ] ] }, - "text": "(\n key \"value\"\n)" + "text": "(\n key \"value\"\n)", + "line": "(o: (key \"value\"))" }, { "name": "two_element_array_is_not_a_single_pair_object", @@ -339,7 +458,8 @@ } ] }, - "text": "(\n \"key\"\n \"value\"\n)" + "text": "(\n \"key\"\n \"value\"\n)", + "line": "(\"key\" \"value\")" }, { "name": "keys_that_need_quoting", @@ -377,7 +497,8 @@ ] ] }, - "text": "(\n \"two words\" 1\n \"\" 2\n \"base64\" 3\n \"with:colon\" 4\n 'with\"quote' 5\n)" + "text": "(\n \"two words\" 1\n \"\" 2\n \"base64\" 3\n \"with:colon\" 4\n 'with\"quote' 5\n)", + "line": "(o: (\"two words\" 1) (\"\" 2) (\"base64\" 3) (\"with:colon\" 4) ('with\"quote' 5))" }, { "name": "base64_key_with_plain_value_is_not_a_marker", @@ -391,7 +512,8 @@ ] ] }, - "text": "(\n \"base64\" \"plain text\"\n)" + "text": "(\n \"base64\" \"plain text\"\n)", + "line": "(o: (\"base64\" \"plain text\"))" }, { "name": "documented_router_state", @@ -437,7 +559,8 @@ ] ] }, - "text": "(\n type \"RouterState\"\n server (\n host \"127.0.0.1\"\n port 18878\n )\n models (\n \"claude-haiku\"\n \"claude-opus\"\n )\n)" + "text": "(\n type \"RouterState\"\n server (\n host \"127.0.0.1\"\n port 18878\n )\n models (\n \"claude-haiku\"\n \"claude-opus\"\n )\n)", + "line": "(o: (type \"RouterState\") (server (o: (host \"127.0.0.1\") (port 18878))) (models (\"claude-haiku\" \"claude-opus\")))" }, { "name": "mixed_types_in_one_object", @@ -496,7 +619,8 @@ "text": "(\n int -7\n float 3.5\n whole_float 2.0\n yes true\n no false\n nothing null\n numeric_string \"18878\"\n boolean_string \"true\"\n)", "skip": { "js": "JavaScript has one number type, so 2.0 and 2 are the same value and the trailing '.0' cannot be recovered when encoding" - } + }, + "line": "(o: (int -7) (float 3.5) (whole_float 2.0) (yes true) (no false) (nothing null) (numeric_string \"18878\") (boolean_string \"true\"))" }, { "name": "only_unwritable_values_are_marked", @@ -522,7 +646,8 @@ ] ] }, - "text": "(\n readable \"still visible\"\n multiline (base64 \"bGluZTEKbGluZTI=\")\n tabbed (base64 \"YQli\")\n)" + "text": "(\n readable \"still visible\"\n multiline (base64 \"bGluZTEKbGluZTI=\")\n tabbed (base64 \"YQli\")\n)", + "line": "(o: (readable \"still visible\") (multiline (base64 \"bGluZTEKbGluZTI=\")) (tabbed (base64 \"YQli\")))" }, { "name": "deeply_nested_objects", @@ -554,7 +679,9 @@ ] ] }, - "text": "(\n a (\n b (\n c (\n 1\n )\n )\n )\n)" + "text": "(\n a (\n b (\n c (\n 1\n )\n )\n )\n)", + "line": "(o: (a (o: (b (o: (c (1)))))))" } - ] + ], + "lineEncoding": "`line` is the single-line form: the same document written without any line break, one record per line. Line breaks no longer separate rows there, so an object names itself with the `o` link id (`(o: (key value) ...)`, `(o:)` when empty) and a bare link is always an array." } diff --git a/js/.changeset/20260827_090000_issue_43_single_line_format.md b/js/.changeset/20260827_090000_issue_43_single_line_format.md new file mode 100644 index 0000000..319f0ce --- /dev/null +++ b/js/.changeset/20260827_090000_issue_43_single_line_format.md @@ -0,0 +1,23 @@ +--- +'lino-objects-codec': minor +--- + +Add `encodeLine` and `decodeLine`: the readable format written on one line, so +an append-only log holds one record per line. Appending is one write, compaction +cuts at a newline, and `grep`, `tail -f` and `wc -l` treat a line as one event. +The output is valid Links Notation, keeps numbers, booleans and `null` bare so +types survive the round trip, and `decode(encodeLine(v))` equals +`decode(encode(v))`. + +The exported `OBJECT_MARKER` (`o`) tells an object from an array on one line: +`(o: (bytes 2827) (complete true))` is a record, `("a" 1)` is a two-element +array, `(o:)` is the empty object and `()` the empty array. Because the marker +is part of the notation, the empty key round-trips as `(o: ("" 2))`. The +single-line spelling of every shared fixture is pinned in +`fixtures/readable-format/cases.json`, so all four languages write the same +bytes. + +Also fixes `decode`, which used to route a readable single-line document such as +`(null 1)` to the compact (base64) reader; the document `(null)` stays the +compact null so older documents keep decoding. See +[issue #43](https://github.com/link-foundation/lino-objects-codec/issues/43). diff --git a/js/README.md b/js/README.md index 62a653d..16f238b 100644 --- a/js/README.md +++ b/js/README.md @@ -27,6 +27,7 @@ These tools enable easy implementation of higher-level features like: - Collections: `Array`, `Object` - Special number values: `NaN`, `Infinity`, `-Infinity` - **Readable by Default**: `encode({ obj })` writes plain, indented text that can be read and reviewed +- **One Record per Line**: `encodeLine({ obj })` writes the same document on one line and `decodeLine({ notation })` reads it back exactly, so an append-only log stays greppable, tailable and countable by `wc -l` - **Object Identity**: Shared references and circular references are preserved by the compact format (`encodeCompact`) via object ids - **Full Unicode**: Strings are written as text; only a value that cannot be written as text (one holding control characters) is base64-encoded, and it is marked individually as `(base64 "…")` - **Opt-in Tracing**: Set `LINO_CODEC_DEBUG=1` to trace encoding and decoding, the same way in every language @@ -108,12 +109,13 @@ console.log(decoded.self === decoded); ## Output Formats -| Function | Output | -| ------------------------------- | ----------------------------------------------- | -| `encode({ obj })` | Readable, indented Links Notation (the default) | -| `encode({ obj, indent: '\t' })` | Same, with a custom indentation string | -| `encodeCompact({ obj })` | The previous single-line, base64 form | -| `encodeObfuscated({ obj })` | Alias of `encodeCompact` | +| Function | Output | +| ------------------------------- | --------------------------------------------------- | +| `encode({ obj })` | Readable, indented Links Notation (the default) | +| `encode({ obj, indent: '\t' })` | Same, with a custom indentation string | +| `encodeLine({ obj })` | The same document on one line, for append-only logs | +| `encodeCompact({ obj })` | The previous single-line, base64 form | +| `encodeObfuscated({ obj })` | Alias of `encodeCompact` | `decode({ notation })` accepts every one of them, so files written by older versions keep working and are rewritten in the readable form the next time they @@ -334,6 +336,25 @@ bare-value lines make an array: base64-encoded on its own and marked as `(base64 "bGluZTEKbGluZTI=")`; everything around it stays readable +### Single-line format (`encodeLine`) + +The same readable document on one line, so an append-only log holds one record +per line — appending is one write, compaction cuts at a newline, and `grep`, +`tail -f` and `wc -l` all treat a line as one event: + +```lino +(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1") (port 18878)))) +``` + +- An object is `(o: (key value) …)` and an empty object is `(o:)` +- An array is `(value …)` and an empty array is `()` +- Scalars and strings are written exactly as in the indented form +- The `o` marker removes the ambiguity a flat layout otherwise has: a bare `( )` + on one line is always an array, so a _hand-written_ `(a 1)` is the two-element + array, not the one-pair object +- `decode({ notation })` reads this form too; `decodeLine({ notation })` is its + exact inverse and rejects input spanning more than one line + ### Compact format (`encodeCompact`) The previous single-line form, kept for compatibility and for the object graphs @@ -441,6 +462,38 @@ Decode Links Notation format to a JavaScript object. - Reconstructed JavaScript object +#### `encodeLine({ obj: obj })` + +Encode a JavaScript object into the readable format on one line. + +**Parameters:** + +- `options.obj` - The JavaScript object to encode + +**Returns:** + +- String representation in readable Links Notation format, holding no newline + +```javascript +encodeLine({ obj: { age: 30 } }); // '(o: (age 30))' +``` + +#### `decodeLine({ notation: notation })` + +Decode one line of a readable Links Notation log. The exact inverse of `encodeLine`. + +**Parameters:** + +- `options.notation` - One line written by `encodeLine` + +**Returns:** + +- Reconstructed JavaScript object + +**Throws:** + +- `SyntaxError` - If the input spans more than one line or is malformed + #### `ObjectCodec` The main codec class that performs encoding and decoding. The module-level `encode({ obj: )` and `decode({ notation: } })` functions use a shared instance of this class. diff --git a/js/examples/append_only_log.js b/js/examples/append_only_log.js new file mode 100644 index 0000000..5a17ad5 --- /dev/null +++ b/js/examples/append_only_log.js @@ -0,0 +1,42 @@ +/** + * Writing an append-only log with one record per line (issue #43). + * + * `encodeLine` keeps a record on one line, so appending is one write, a + * compactor can cut the file at any newline, and `grep`, `tail -f` and `wc -l` + * all treat one line as one event. `decodeLine` reads a line back exactly. + */ + +import assert from 'node:assert'; +import { encodeLine, decodeLine } from '../src/index.js'; + +const record = (phase, bytes, complete) => ({ phase, bytes, complete }); + +function main() { + console.log('=== Append-only log, one record per line ===\n'); + + // Appending: each record becomes exactly one line of the file. + const entries = [ + record('stream_start', 0, false), + record('stream_chunk', 1024, false), + record('stream_end', 2827, true), + ]; + const log = entries + .map((entry) => `${encodeLine({ obj: entry })}\n`) + .join(''); + process.stdout.write(log); + + // Counting: one line is one event, so `wc -l` answers how many there were. + const lines = log.trimEnd().split('\n'); + console.log(`\nrecords: ${lines.length}`); + + // Reading: a line reader hands over one record at a time. + const decoded = decodeLine({ notation: lines[lines.length - 1] }); + console.log(`last record: ${JSON.stringify(decoded)}`); + assert.deepStrictEqual(decoded, record('stream_end', 2827, true)); + + // Filtering: the text stays readable, so plain string tools still work. + const finished = lines.filter((line) => line.includes('(complete true)')); + console.log(`finished records: ${finished.length}`); +} + +main(); diff --git a/js/src/codec.js b/js/src/codec.js index d457d42..520d0a6 100644 --- a/js/src/codec.js +++ b/js/src/codec.js @@ -99,6 +99,39 @@ export class ObjectCodec { return readable.encode(obj, indent); } + /** + * Encode a JavaScript object to the readable, single-line Links Notation + * format. + * + * The result never contains a newline, so one value is one line: an + * append-only log written this way stays greppable, tailable and countable by + * `wc -l`. See `readable.js` for the exact shape. + * + * @param {Object} options - Options + * @param {*} options.obj - The JavaScript object to encode + * @returns {string} One line of readable Links Notation + */ + encodeLine(options = {}) { + const { obj } = options; + return readable.encodeLine(obj); + } + + /** + * Decode one line of the readable, single-line Links Notation format. + * + * This is the exact inverse of {@link ObjectCodec#encodeLine}. Input spanning + * more than one line is rejected, so two log records never merge into one + * value. + * + * @param {Object} options - Options + * @param {string} options.notation - One line of readable Links Notation + * @returns {*} Reconstructed JavaScript object + */ + decodeLine(options = {}) { + const { notation } = options; + return readable.decodeLine(notation); + } + /** * Encode a JavaScript object to the compact, single-line Links Notation format. * @@ -605,6 +638,16 @@ const COMPACT_TYPE_MARKERS = new Set([ 'dict', ]); +/** + * Markers a compact document writes without a payload, so `(null)` is a compact + * null while `(null 1)` is a readable line holding two values. + */ +const EMPTY_BODY_MARKERS = new Set([ + ObjectCodec.TYPE_NULL, + ObjectCodec.TYPE_UNDEFINED, + 'None', +]); + /** * Whether a document is in the compact (base64) format rather than the readable * one. The compact format always opens with `(` followed by a type marker, @@ -623,28 +666,40 @@ export function isCompactNotation(notation) { return false; } - const tokens = firstLine - .slice(1) - .split(/[\s()]+/) - .filter((token) => token.length > 0); - - let marker = tokens[0]; - if (marker === undefined) { - return false; - } + // A compact document names the type of its value first, so a link that opens + // another link straight away is the readable form, whose links nest. + let [marker, rest] = splitToken(firstLine.slice(1).trimStart()); // Skip the `obj_N:` definition id, if present. if (marker.endsWith(':')) { if (!marker.startsWith('obj_')) { return false; } - marker = tokens[1]; - if (marker === undefined) { - return false; - } + [marker, rest] = splitToken(rest.trimStart()); + } + + if (!COMPACT_TYPE_MARKERS.has(marker)) { + return false; + } + + // A compact null is the whole link: `(null)`. A link that holds more than the + // marker is a readable line whose first value happens to be null. + if (EMPTY_BODY_MARKERS.has(marker)) { + return rest.trimStart().startsWith(')'); } - return COMPACT_TYPE_MARKERS.has(marker); + return true; +} + +/** + * Split off the first token of a link body: the text up to the next whitespace + * or parenthesis. A body that opens with a parenthesis has no token of its own. + * @param {string} input - The link body + * @returns {[string, string]} The token and the text after it + */ +function splitToken(input) { + const end = input.search(/[\s()]/); + return end === -1 ? [input, ''] : [input.slice(0, end), input.slice(end)]; } // Convenience functions @@ -661,6 +716,33 @@ export function encode(options = {}) { return _defaultCodec.encode(options); } +/** + * Encode a JavaScript object to the readable, single-line Links Notation format. + * + * The result never contains a newline, so one value is one line of an + * append-only log. + * + * @param {Object} options - Options + * @param {*} options.obj - The JavaScript object to encode + * @returns {string} One line of readable Links Notation + */ +export function encodeLine(options = {}) { + return _defaultCodec.encodeLine(options); +} + +/** + * Decode one line of the readable, single-line Links Notation format. + * + * The exact inverse of {@link encodeLine}. + * + * @param {Object} options - Options + * @param {string} options.notation - One line of readable Links Notation + * @returns {*} Reconstructed JavaScript object + */ +export function decodeLine(options = {}) { + return _defaultCodec.decodeLine(options); +} + /** * Encode a JavaScript object to the compact, single-line Links Notation format. * diff --git a/js/src/index.js b/js/src/index.js index ccbf01c..d7a99bd 100644 --- a/js/src/index.js +++ b/js/src/index.js @@ -3,6 +3,7 @@ * * This library provides: * - Readable recursive indented Links Notation for JSON-style repository data + * - Readable single-line Links Notation for append-only logs, one record per line * - Typed serialization/deserialization for exact JavaScript object graphs * - Typed support for circular references and shared object identity * - JSON to Links Notation conversion utilities @@ -20,6 +21,8 @@ export { ObjectCodec, encode, + encodeLine, + decodeLine, encodeCompact, encodeObfuscated, decode, @@ -31,6 +34,7 @@ export { export { DEFAULT_INDENT, BASE64_MARKER, + OBJECT_MARKER, CircularReferenceError, } from './readable.js'; diff --git a/js/src/readable.js b/js/src/readable.js index 9be5b6b..c5d3649 100644 --- a/js/src/readable.js +++ b/js/src/readable.js @@ -42,6 +42,34 @@ * CRLF normalisation would corrupt) are marked individually as * `(base64 "…")` instead of encoding the whole document. * + * # Single-line form + * + * {@link encodeLine} writes the same document on one line, so one record is one + * line and an append-only log stays greppable, tailable and countable by + * `wc -l`. Rows can no longer be told apart by line breaks there, so an object + * names itself with the `o` link id the notation already has, and its pairs are + * written as their own links: + * + * ```text + * (o: (type "RouterState") (server (o: (host "127.0.0.1") (port 18878))) (models ("claude-haiku" "claude-opus"))) + * ``` + * + * | Value | Single-line form | + * | ---------------- | ------------------------------- | + * | plain object | `(o: (key value) …)` | + * | empty object | `(o:)` | + * | `Array` | `(value …)` | + * | empty `Array` | `()` | + * | scalars | exactly as in the indented form | + * + * The marker is what answers the ambiguity a flat layout otherwise has: without + * it `((key value))` reads both as the one-pair object and as the array holding + * the two-element array, and an empty key makes it worse. With it, a bare `( )` + * is always an array and a marked one is always an object, so every value — + * empty key included — survives the round trip. Consequently a *hand-written* + * one-line link such as `(a 1)` is the two-element array, not the one-pair + * object: on one line, objects say so. + * * @module readable */ @@ -53,6 +81,9 @@ export const DEFAULT_INDENT = ' '; /** Marker used for values that cannot be represented as plain text. */ export const BASE64_MARKER = 'base64'; +/** Link id naming an object in the single-line form, written as `(o: …)`. */ +export const OBJECT_MARKER = 'o'; + /** Literals that a bare reference decodes to instead of a string. */ const BARE_LITERALS = new Map([ ['null', null], @@ -99,6 +130,61 @@ export function encode(value, indent = DEFAULT_INDENT) { return out.join(''); } +/** + * Encode a value into the readable, single-line Links Notation form. + * + * The result never contains a newline, so one value is one line of an + * append-only log. See the module documentation for the shape. + * @param {*} value - The value to encode + * @returns {string} The readable Links Notation document, on one line + * @throws {CircularReferenceError} If the value refers back to itself + * @throws {TypeError} If the value holds a type this format cannot write + */ +export function encodeLine(value) { + const out = []; + writeLineValue(value, out, new Set()); + return out.join(''); +} + +/** + * Strip the line breaks framing a record, without a regular expression. + * + * A regular expression anchored at the end backtracks over a run of newlines, + * so a long run of them costs more than linear time. Scanning from both ends + * costs one pass over the framing characters. + * @param {string} text - The text to trim + * @returns {string} The text without leading or trailing newlines + */ +function trimLineBreaks(text) { + let start = 0; + let end = text.length; + while (start < end && (text[start] === '\n' || text[start] === '\r')) { + start += 1; + } + while (end > start && (text[end - 1] === '\n' || text[end - 1] === '\r')) { + end -= 1; + } + return text.slice(start, end); +} + +/** + * Decode the readable, single-line Links Notation form back into a value. + * + * This is the exact inverse of {@link encodeLine}. Input spanning more than one + * line is rejected: a line-based reader hands over one record at a time, and + * silently accepting several would merge two records into one value. + * @param {string} text - One line of a readable Links Notation document + * @returns {*} The reconstructed value + * @throws {SyntaxError} If the input holds more than one line + */ +export function decodeLine(text) { + const line = trimLineBreaks(text); + if (line.includes('\n') || line.includes('\r')) { + throw new SyntaxError('a single-line document cannot contain a line break'); + } + return decode(line); +} + /** * Decode the readable, indented Links Notation form back into a value. * @param {string} text - The readable Links Notation document @@ -119,7 +205,7 @@ export function decode(text) { return nodeToValue(rows[0][0]); } - return rowsToValue(rows, true); + return rowsToValue(rows, true, false); } // === Encoding === @@ -157,6 +243,51 @@ function writeValue(value, indent, level, out, path) { out.push(formatScalar(value)); } +/** + * Write a value on one line. Objects name themselves with the `o` link id and + * write each pair as its own link, so nothing depends on where lines break. + * @param {*} value - The value to write + * @param {string[]} out - Output chunks, appended in place + * @param {Set} path - Containers currently being written + */ +function writeLineValue(value, out, path) { + if (Array.isArray(value)) { + enterPath(value, path); + out.push('('); + value.forEach((item, index) => { + if (index > 0) { + out.push(' '); + } + writeLineValue(item, out, path); + }); + out.push(')'); + path.delete(value); + return; + } + + if (isPlainContainer(value)) { + enterPath(value, path); + const entries = Object.entries(value); + if (entries.length === 0) { + // `()` is the empty array, so the empty object keeps its marker. + out.push(`(${OBJECT_MARKER}:)`); + path.delete(value); + return; + } + out.push(`(${OBJECT_MARKER}:`); + for (const [key, child] of entries) { + out.push(` (${formatKey(key)} `); + writeLineValue(child, out, path); + out.push(')'); + } + out.push(')'); + path.delete(value); + return; + } + + out.push(formatScalar(value)); +} + /** * Mark a container as being written, so a reference back to it is caught. * @@ -471,14 +602,33 @@ class Cursor { if (token.kind === TOKEN_OPEN) { this.pos += 1; + const object = this.takeObjectMarker(); const multiline = this.linkIsMultiline(); const rows = this.parseRows(false); - return { ref: false, rows, multiline }; + return { ref: false, rows, multiline, object }; } throw new SyntaxError('unexpected token in readable notation'); } + /** + * Consume the `o:` marker if the link that just opened carries one, which is + * how the single-line form says "this link is an object, not an array". + * @returns {boolean} True when the marker was there and was consumed + */ + takeObjectMarker() { + const token = this.tokens[this.pos]; + const isMarker = + token !== undefined && + token.kind === TOKEN_REF && + !token.quoted && + token.value === `${OBJECT_MARKER}:`; + if (isMarker) { + this.pos += 1; + } + return isMarker; + } + /** * Whether the link that just opened spans more than one line, which is what * tells an empty object (`(\n)`) from an empty array (`()`). @@ -500,10 +650,14 @@ class Cursor { function nodeToValue(node) { return node.ref ? refToValue(node.value, node.quoted) - : rowsToValue(node.rows, node.multiline); + : rowsToValue(node.rows, node.multiline, node.object); } -function rowsToValue(rows, multiline) { +function rowsToValue(rows, multiline, objectMarker) { + if (objectMarker) { + return markedObjectToValue(rows); + } + if (rows.length === 0) { return multiline ? {} : []; } @@ -513,6 +667,12 @@ function rowsToValue(rows, multiline) { return marked.value; } + // Written on one line, a link is a list of values: an object on one line says + // so with the `o:` marker, which is what keeps `(key value)` unambiguous. + if (!multiline) { + return rows.flatMap((row) => row.map(nodeToValue)); + } + // `key value` on every line makes an object; anything else is a list of values. const isObject = rows.every((row) => row.length === 2 && row[0].ref); @@ -533,6 +693,42 @@ function rowsToValue(rows, multiline) { return items; } +/** + * Build the object a `(o: (key value) …)` link describes. Every value in it is + * a pair, so anything else is a malformed document rather than a silent array. + * @param {Array>} rows - The rows of the marked link + * @returns {object} The reconstructed object + * @throws {SyntaxError} If the link holds anything that is not a pair + */ +function markedObjectToValue(rows) { + const result = {}; + + for (const node of rows.flat()) { + if (node.ref || node.object) { + throw new SyntaxError( + `an object marked '${OBJECT_MARKER}:' holds (key value) pairs, ` + + 'found a value that is not a pair' + ); + } + if (node.rows.length !== 1) { + throw new SyntaxError( + `an object marked '${OBJECT_MARKER}:' holds (key value) pairs, ` + + `found a link of ${node.rows.length} lines` + ); + } + const [row] = node.rows; + if (row.length !== 2 || !row[0].ref) { + throw new SyntaxError( + `an object marked '${OBJECT_MARKER}:' holds (key value) pairs, ` + + `found a link of ${row.length} values` + ); + } + result[row[0].value] = nodeToValue(row[1]); + } + + return result; +} + /** * Recognise `(base64 "…")`, the individual marker for values that could not be * written as text. A quoted `base64` key is an ordinary object key, not a marker. diff --git a/js/tests/test_readable_conformance.test.js b/js/tests/test_readable_conformance.test.js index 5b96740..8a976a1 100644 --- a/js/tests/test_readable_conformance.test.js +++ b/js/tests/test_readable_conformance.test.js @@ -3,8 +3,9 @@ * * The cases live in `fixtures/readable-format/cases.json` at the repository root * and are shared by the JavaScript, Python, Rust and C# suites: every - * implementation has to encode the same value to exactly the same text, which is - * what keeps the four outputs byte-identical. + * implementation has to encode the same value to exactly the same text and to + * exactly the same single line, which is what keeps the four outputs + * byte-identical. */ import { test } from 'node:test'; @@ -12,7 +13,7 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; -import { encode, decode } from '../src/index.js'; +import { encode, encodeLine, decode, decodeLine } from '../src/index.js'; const LANGUAGE = 'js'; const LANGUAGES = new Set(['js', 'python', 'rust', 'csharp']); @@ -120,4 +121,26 @@ for (const testCase of cases) { `${JSON.stringify(decode({ notation: testCase.text }))} != ${JSON.stringify(build(testCase.value))}` ); }); + + test(`encodeLine matches the shared line: ${testCase.name}`, () => { + assert.equal(encodeLine({ obj: build(testCase.value) }), testCase.line); + }); + + test(`the shared line holds no line break: ${testCase.name}`, () => { + assert.ok(!/[\n\r]/.test(testCase.line), testCase.line); + }); + + test(`decodeLine matches the shared value: ${testCase.name}`, () => { + assert.ok( + same(decodeLine({ notation: testCase.line }), build(testCase.value)), + `${JSON.stringify(decodeLine({ notation: testCase.line }))} != ${JSON.stringify(build(testCase.value))}` + ); + }); + + test(`decode reads the shared line too: ${testCase.name}`, () => { + assert.ok( + same(decode({ notation: testCase.line }), build(testCase.value)), + `${JSON.stringify(decode({ notation: testCase.line }))} != ${JSON.stringify(build(testCase.value))}` + ); + }); } diff --git a/js/tests/test_single_line_format.test.js b/js/tests/test_single_line_format.test.js new file mode 100644 index 0000000..8490c67 --- /dev/null +++ b/js/tests/test_single_line_format.test.js @@ -0,0 +1,128 @@ +/** + * Tests for the readable, single-line format produced by `encodeLine()` + * (issue #43). + * + * An append-only log wants one record per line: appending is one write, + * compaction cuts at a newline, and `grep`, `tail -f` and `wc -l` all treat a + * line as an event. `encode()` spreads a record over many lines and + * `encodeCompact()` hides it in base64, so neither serves that reader. + */ + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { Parser } from 'links-notation'; +import { encode, encodeLine, decode, decodeLine } from '../src/index.js'; + +/** A record of the shape an append-only log actually holds. */ +const LOG_RECORD = { + bytes: 2827, + complete: true, + server: { host: '127.0.0.1', port: 18878 }, + models: ['claude-haiku', 'claude-opus'], +}; + +const LOG_RECORD_LINE = + '(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1")' + + ' (port 18878))) (models ("claude-haiku" "claude-opus")))'; + +test('a record is written on one line', () => { + const line = encodeLine({ obj: LOG_RECORD }); + assert.ok(!/[\n\r]/.test(line), line); + assert.equal(line, LOG_RECORD_LINE); +}); + +test('a line is valid links notation', () => { + new Parser().parse(encodeLine({ obj: LOG_RECORD })); +}); + +test('the hand-rolled dialect is the one the parser rejects', () => { + assert.throws(() => + new Parser().parse('((:"bytes" 2827) (:"complete" true))') + ); +}); + +test('both forms of the same value decode alike', () => { + const values = [ + LOG_RECORD, + [], + {}, + [{}, []], + { empty: [] }, + 42, + null, + 'text', + ]; + for (const value of values) { + assert.deepEqual( + decode({ notation: encodeLine({ obj: value }) }), + decode({ notation: encode({ obj: value }) }) + ); + assert.deepEqual( + decodeLine({ notation: encodeLine({ obj: value }) }), + value + ); + } +}); + +test('a string keeps its own characters on one line', () => { + const value = { text: 'quote " backslash \\ ünïcödé' }; + const line = encodeLine({ obj: value }); + assert.equal(line, `(o: (text 'quote " backslash \\ ünïcödé'))`); + assert.deepEqual(decodeLine({ notation: line }), value); +}); + +test('a string holding a newline still fits on one line', () => { + const value = { readable: 'still visible', multiline: 'line1\nline2' }; + const line = encodeLine({ obj: value }); + assert.equal( + line, + '(o: (readable "still visible") (multiline (base64 "bGluZTEKbGluZTI=")))' + ); + assert.ok(!/[\n\r]/.test(line), line); + assert.deepEqual(decodeLine({ notation: line }), value); +}); + +test('a one-pair object is not a two-element array', () => { + assert.equal(encodeLine({ obj: { a: 1 } }), '(o: (a 1))'); + assert.equal(encodeLine({ obj: ['a', 1] }), '("a" 1)'); + assert.deepEqual(decodeLine({ notation: '(o: (a 1))' }), { a: 1 }); + assert.deepEqual(decodeLine({ notation: '("a" 1)' }), ['a', 1]); +}); + +test('the empty key survives the round trip', () => { + const value = { '': 2 }; + assert.equal(encodeLine({ obj: value }), `(o: ("" 2))`); + assert.deepEqual(decodeLine({ notation: encodeLine({ obj: value }) }), value); +}); + +test('a marked object holding something that is not a pair is rejected', () => { + assert.throws(() => decodeLine({ notation: '(o: 1 2)' }), /pairs/); +}); + +test('several lines are not one record', () => { + assert.throws(() => decodeLine({ notation: '(o: (a 1))\n(o: (b 2))' })); +}); + +test('a trailing newline is not a second record', () => { + assert.deepEqual(decodeLine({ notation: '(o: (a 1))\n' }), { a: 1 }); +}); + +test('a line starting with null is still read as a line', () => { + assert.deepEqual(decode({ notation: '(null 1)' }), [null, 1]); + assert.deepEqual(decode({ notation: '(o: (a null))' }), { a: null }); + // The one document both forms claim: `(null)` is the compact null, and stays + // read that way, so documents written before this format keep decoding. + assert.equal(decode({ notation: '(null)' }), null); +}); + +test('a long run of line breaks is rejected without a slowdown', () => { + // CodeQL alert js/polynomial-redos: trimming the framing newlines with + // `/[\n\r]+$/` backtracked once per newline, so a record followed by a long + // run of them cost quadratic time. The scan that replaced it is linear, and + // the input is still refused for holding more than one line. + const notation = `${LOG_RECORD_LINE}${'\n'.repeat(200000)}x`; + const started = process.hrtime.bigint(); + assert.throws(() => decodeLine({ notation }), SyntaxError); + const elapsedMs = Number(process.hrtime.bigint() - started) / 1e6; + assert.ok(elapsedMs < 2000, `took ${elapsedMs}ms`); +}); diff --git a/python/README.md b/python/README.md index f4646c3..2539018 100644 --- a/python/README.md +++ b/python/README.md @@ -10,6 +10,7 @@ A Python library to encode/decode objects to/from Links Notation format. This li ## Features - **Readable by Default**: `encode()` writes plain, indented text that can be read and reviewed +- **One Record per Line**: `encode_line()` writes the same document on one line and `decode_line()` reads it back exactly, so an append-only log stays greppable, tailable and countable by `wc -l` - **Universal Serialization**: Encode Python objects to Links Notation format - **Type Support**: Handle all common Python types: - Basic types: `None`, `bool`, `int`, `float`, `str` @@ -120,6 +121,7 @@ assert decode(encode(complex_data)) == complex_data | --- | --- | | `encode(obj)` | Readable, indented Links Notation (the default) | | `encode(obj, indent="\t")` | Same, with a custom indentation string | +| `encode_line(obj)` | The same readable document on one line, for append-only logs | | `encode_compact(obj)` | The previous single-line, base64 form | | `encode_obfuscated(obj)` | Alias of `encode_compact` | @@ -192,6 +194,25 @@ list: base64-encoded on its own and marked as `(base64 "bGluZTEKbGluZTI=")`; everything around it stays readable +### Single-line format (`encode_line`) + +The same readable document on one line, so an append-only log holds one record +per line — appending is one write, compaction cuts at a newline, and `grep`, +`tail -f` and `wc -l` all treat a line as one event: + +```lino +(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1") (port 18878)))) +``` + +- A dict is `(o: (key value) …)` and an empty dict is `(o:)` +- A list is `(value …)` and an empty list is `()` +- Scalars and strings are written exactly as in the indented form +- The `o` marker removes the ambiguity a flat layout otherwise has: a bare `( )` + on one line is always a list, so a *hand-written* `(a 1)` is the two-element + list, not the one-pair dict +- `decode()` reads this form too; `decode_line()` is its exact inverse and + rejects input spanning more than one line + ### Compact format (`encode_compact`) The previous single-line form, kept for compatibility and for the object graphs @@ -253,6 +274,36 @@ Decode Links Notation format to a Python object. **Returns:** - Reconstructed Python object +### `encode_line(obj: Any) -> str` + +Encode a Python object into the readable format on one line. + +**Parameters:** +- `obj`: The Python object to encode + +**Returns:** +- String representation in readable Links Notation format, holding no newline + +```python +>>> from link_notation_objects_codec import encode_line +>>> encode_line({"age": 30}) +'(o: (age 30))' +``` + +### `decode_line(notation: str) -> Any` + +Decode one line of a readable Links Notation log. The exact inverse of +`encode_line()`. + +**Parameters:** +- `notation`: One line written by `encode_line()` + +**Returns:** +- Reconstructed Python object + +**Raises:** +- `ReadableFormatError`: If the input spans more than one line or is malformed + ### `ObjectCodec` The main codec class that performs encoding and decoding. The module-level `encode()` and `decode()` functions use a shared instance of this class. diff --git a/python/changelog.d/20260827_090000_issue_43_single_line_format.md b/python/changelog.d/20260827_090000_issue_43_single_line_format.md new file mode 100644 index 0000000..33d769f --- /dev/null +++ b/python/changelog.d/20260827_090000_issue_43_single_line_format.md @@ -0,0 +1,24 @@ +### Added + +- `encode_line` and `decode_line`: the readable format written on one line, so + an append-only log holds one record per line. Appending is one write, + compaction cuts at a newline, and `grep`, `tail -f` and `wc -l` treat a line + as one event. The output is valid Links Notation, keeps numbers, booleans and + `None` bare so types survive the round trip, and `decode(encode_line(v))` + equals `decode(encode(v))`. See + [issue #43](https://github.com/link-foundation/lino-objects-codec/issues/43). +- `OBJECT_MARKER`, the `o` link id that tells an object from an array on one + line: `(o: (bytes 2827) (complete true))` is a record, `("a" 1)` is a + two-element array, `(o:)` is the empty object and `()` the empty array. The + marker is part of the notation, so the empty key round-trips as `(o: ("" 2))`. + The single-line spelling of every shared fixture is pinned in + `fixtures/readable-format/cases.json`, so all four languages write the same + bytes. +- The `examples/append_only_log.py` example, which writes, counts, greps and + reads back a log of one record per line. + +### Fixed + +- `decode` no longer routes a readable single-line document such as `(None 1)` + to the compact (base64) reader. Only a compact document keeps that path; the + document `(None)` stays the compact `None` so older documents keep decoding. diff --git a/python/examples/append_only_log.py b/python/examples/append_only_log.py new file mode 100644 index 0000000..4955546 --- /dev/null +++ b/python/examples/append_only_log.py @@ -0,0 +1,44 @@ +"""Writing an append-only log with one record per line (issue #43). + +``encode_line`` keeps a record on one line, so appending is one write, a +compactor can cut the file at any newline, and ``grep``, ``tail -f`` and +``wc -l`` all treat one line as one event. ``decode_line`` reads a line back +exactly. +""" + +from link_notation_objects_codec import decode_line, encode_line + + +def record(phase: str, num_bytes: int, complete: bool) -> dict: + """Build a record of the shape an append-only log actually holds.""" + return {"phase": phase, "bytes": num_bytes, "complete": complete} + + +def main() -> None: + print("=== Append-only log, one record per line ===\n") + + # Appending: each record becomes exactly one line of the file. + entries = [ + record("stream_start", 0, False), + record("stream_chunk", 1024, False), + record("stream_end", 2827, True), + ] + log = "".join(f"{encode_line(entry)}\n" for entry in entries) + print(log, end="") + + # Counting: one line is one event, so `wc -l` answers how many there were. + lines = log.splitlines() + print(f"\nrecords: {len(lines)}") + + # Reading: a line reader hands over one record at a time. + decoded = decode_line(lines[-1]) + print(f"last record: {decoded}") + assert decoded == record("stream_end", 2827, True) + + # Filtering: the text stays readable, so plain string tools still work. + finished = [line for line in lines if "(complete true)" in line] + print(f"finished records: {len(finished)}") + + +if __name__ == "__main__": + main() diff --git a/python/src/link_notation_objects_codec/__init__.py b/python/src/link_notation_objects_codec/__init__.py index dfc00f7..9a40c58 100644 --- a/python/src/link_notation_objects_codec/__init__.py +++ b/python/src/link_notation_objects_codec/__init__.py @@ -6,6 +6,8 @@ :func:`encode` writes the readable, indented format by default; :func:`decode` reads both that and the compact (base64) format written by earlier versions. +:func:`encode_line` writes the same readable document on one line, so an +append-only log holds one record per line. """ from importlib.metadata import PackageNotFoundError @@ -15,8 +17,10 @@ ObjectCodec, decode, decode_compact, + decode_line, encode, encode_compact, + encode_line, encode_obfuscated, is_compact_notation, ) @@ -30,6 +34,7 @@ from .readable import ( BASE64_MARKER, DEFAULT_INDENT, + OBJECT_MARKER, CircularReferenceError, ReadableFormatError, ) @@ -43,9 +48,11 @@ __all__ = [ "ObjectCodec", "encode", + "encode_line", "encode_compact", "encode_obfuscated", "decode", + "decode_line", "decode_compact", "is_compact_notation", "escape_reference", @@ -54,6 +61,7 @@ "parse_indented", "DEFAULT_INDENT", "BASE64_MARKER", + "OBJECT_MARKER", "ReadableFormatError", "CircularReferenceError", "DEBUG_ENV_VAR", diff --git a/python/src/link_notation_objects_codec/codec.py b/python/src/link_notation_objects_codec/codec.py index 9e76fb1..12e5878 100644 --- a/python/src/link_notation_objects_codec/codec.py +++ b/python/src/link_notation_objects_codec/codec.py @@ -44,6 +44,11 @@ ) +#: Markers a compact document writes without a payload, so ``(null)`` is a +#: compact null while ``(null 1)`` is a readable line holding two values. +_EMPTY_BODY_MARKERS: frozenset[str] = frozenset({"null", "None", "undefined"}) + + def is_compact_notation(notation: str) -> bool: """Whether a document is in the compact (type-tagged, base64) format. @@ -62,21 +67,35 @@ def is_compact_notation(notation: str) -> bool: if first_line is None or not first_line.startswith("("): return False - tokens = [token for token in re.split(r"[\s()]+", first_line[1:]) if token] - if not tokens: - return False - - marker = tokens[0] + # A compact document names the type of its value first, so a link that opens + # another link straight away is the readable form, whose links nest. + marker, rest = _split_token(first_line[1:].lstrip()) # Skip the ``obj_N:`` definition id, if present. if marker.endswith(":"): if not marker[:-1].startswith("obj_"): return False - if len(tokens) < 2: - return False - marker = tokens[1] + marker, rest = _split_token(rest.lstrip()) + + if marker not in _COMPACT_TYPE_MARKERS: + return False - return marker in _COMPACT_TYPE_MARKERS + # A compact null is the whole link: ``(null)``. A link that holds more than + # the marker is a readable line whose first value happens to be null. + if marker in _EMPTY_BODY_MARKERS: + return rest.lstrip().startswith(")") + + return True + + +def _split_token(text: str) -> tuple[str, str]: + """Split off the first token of a link body. + + The token is the text up to the next whitespace or parenthesis; a body that + opens with a parenthesis has no token of its own. + """ + match = re.search(r"[\s()]", text) + return (text, "") if match is None else (text[: match.start()], text[match.start() :]) class ObjectCodec: @@ -183,6 +202,37 @@ def encode(self, obj: Any, indent: str = readable.DEFAULT_INDENT) -> str: """ return readable.encode(obj, indent) + def encode_line(self, obj: Any) -> str: + """ + Encode a Python object to the readable, single-line Links Notation format. + + The result never contains a newline, so one value is one line: an + append-only log written this way stays greppable, tailable and countable + by ``wc -l``. See :mod:`link_notation_objects_codec.readable` for the shape. + + Args: + obj: The Python object to encode + + Returns: + One line of readable Links Notation + """ + return readable.encode_line(obj) + + def decode_line(self, notation: str) -> Any: + """ + Decode one line of the readable, single-line Links Notation format. + + This is the exact inverse of :meth:`encode_line`. Input spanning more + than one line is rejected, so two log records never merge into one value. + + Args: + notation: One line of readable Links Notation + + Returns: + Reconstructed Python object + """ + return readable.decode_line(notation) + def encode_compact(self, obj: Any) -> str: """ Encode a Python object to the compact, single-line Links Notation format. @@ -589,6 +639,45 @@ def encode(obj: Any, indent: str = readable.DEFAULT_INDENT) -> str: return _default_codec.encode(obj, indent) +def encode_line(obj: Any) -> str: + """ + Encode a Python object to the readable, single-line Links Notation format. + + The result never contains a newline, so one value is one line of an + append-only log. + + Args: + obj: The Python object to encode + + Returns: + One line of readable Links Notation + + Example: + >>> encode_line({"age": 30}) + '(o: (age 30))' + """ + return _default_codec.encode_line(obj) + + +def decode_line(notation: str) -> Any: + """ + Decode one line of the readable, single-line Links Notation format. + + The exact inverse of :func:`encode_line`. + + Args: + notation: One line of readable Links Notation + + Returns: + Reconstructed Python object + + Example: + >>> decode_line('(o: (age 30))') + {'age': 30} + """ + return _default_codec.decode_line(notation) + + def encode_compact(obj: Any) -> str: """ Encode a Python object to the compact, single-line Links Notation format. diff --git a/python/src/link_notation_objects_codec/readable.py b/python/src/link_notation_objects_codec/readable.py index 19bfa35..16f1898 100644 --- a/python/src/link_notation_objects_codec/readable.py +++ b/python/src/link_notation_objects_codec/readable.py @@ -42,6 +42,35 @@ control characters (including newlines and tabs, which line-based tooling and CRLF normalisation would corrupt) are marked individually as ``(base64 "...")`` instead of encoding the whole document. + +Single-line form +---------------- + +:func:`encode_line` writes the same document on one line, so one record is one +line and an append-only log stays greppable, tailable and countable by ``wc -l``. +Rows can no longer be told apart by line breaks there, so a dict names itself +with the ``o`` link id the notation already has, and its pairs are written as +their own links:: + + (o: (type "RouterState") (server (o: (host "127.0.0.1") (port 18878)))) + +================== =============================== +Value Single-line form +================== =============================== +``dict`` ``(o: (key value) ...)`` +empty ``dict`` ``(o:)`` +``list`` ``(value ...)`` +empty ``list`` ``()`` +scalars exactly as in the indented form +================== =============================== + +The marker is what answers the ambiguity a flat layout otherwise has: without it +``((key value))`` reads both as the one-pair dict and as the list holding the +two-element list, and an empty key makes it worse. With it, a bare ``( )`` is +always a list and a marked one is always a dict, so every value -- empty key +included -- survives the round trip. Consequently a *hand-written* one-line link +such as ``(a 1)`` is the two-element list, not the one-pair dict: on one line, +dicts say so. """ import base64 @@ -60,6 +89,9 @@ #: Marker used for values that cannot be represented as plain text. BASE64_MARKER = "base64" +#: Link id naming a dict in the single-line form, written as ``(o: ...)``. +OBJECT_MARKER = "o" + #: Quote characters that open a quoted reference. _QUOTE_CHARS = ("'", '"', "`") @@ -102,6 +134,49 @@ def encode(value: Any, indent: str = DEFAULT_INDENT) -> str: return "".join(out) +def encode_line(value: Any) -> str: + """Encode a value into the readable, single-line Links Notation form. + + The result never contains a newline, so one value is one line of an + append-only log. See the module documentation for the shape. + + Args: + value: The value to encode. + + Returns: + The readable Links Notation document, on one line. + + Raises: + CircularReferenceError: If the value refers back to itself. + TypeError: If the value holds a type this format cannot write. + """ + out: list[str] = [] + _write_line_value(value, out, set()) + return "".join(out) + + +def decode_line(text: str) -> Any: + """Decode the readable, single-line Links Notation form back into a value. + + This is the exact inverse of :func:`encode_line`. Input spanning more than + one line is rejected: a line-based reader hands over one record at a time, + and silently accepting several would merge two records into one value. + + Args: + text: One line of a readable Links Notation document. + + Returns: + The reconstructed value. + + Raises: + ReadableFormatError: If the input holds more than one line. + """ + line = text.strip("\n\r") + if "\n" in line or "\r" in line: + raise ReadableFormatError("a single-line document cannot contain a line break") + return decode(line) + + def decode(text: str) -> Any: """Decode the readable, indented Links Notation form back into a value. @@ -126,7 +201,7 @@ def decode(text: str) -> Any: if len(rows) == 1 and len(rows[0]) == 1: return _node_to_value(rows[0][0]) - return _rows_to_value(rows, multiline=True) + return _rows_to_value(rows, multiline=True, object_marker=False) # === Encoding === @@ -165,6 +240,41 @@ def write_item(item: Any) -> None: out.append(_format_scalar(value)) +def _write_line_value(value: Any, out: list[str], path: set[int]) -> None: + """Write a value on one line. + + Dicts name themselves with the ``o`` link id and write each pair as its own + link, so nothing depends on where lines break. + """ + if isinstance(value, dict): + with _on_path(value, path): + items = list(value.items()) + if not items: + # ``()`` is the empty list, so the empty dict keeps its marker. + out.append(f"({OBJECT_MARKER}:)") + return + + out.append(f"({OBJECT_MARKER}:") + for key, child in items: + out.append(f" ({_format_key(key)} ") + _write_line_value(child, out, path) + out.append(")") + out.append(")") + return + + if isinstance(value, (list, tuple, set, frozenset)): + with _on_path(value, path): + out.append("(") + for index, item in enumerate(value): + if index: + out.append(" ") + _write_line_value(item, out, path) + out.append(")") + return + + out.append(_format_scalar(value)) + + @contextmanager def _on_path(value: Any, path: set[int]) -> Iterator[None]: """Mark a container as being written, so a reference back to it is caught. @@ -313,7 +423,7 @@ class _Node: """A parsed element: a reference (remembering whether it was quoted, which is what distinguishes a string from a number) or a link.""" - __slots__ = ("is_ref", "value", "quoted", "rows", "multiline") + __slots__ = ("is_ref", "value", "quoted", "rows", "multiline", "is_object") def __init__( self, @@ -322,12 +432,14 @@ def __init__( quoted: bool = False, rows: list[list["_Node"]] | None = None, multiline: bool = False, + is_object: bool = False, ) -> None: self.is_ref = is_ref self.value = value self.quoted = quoted self.rows = rows if rows is not None else [] self.multiline = multiline + self.is_object = is_object def _tokenize(text: str) -> list[_Token]: @@ -438,12 +550,26 @@ def parse_node(self) -> _Node: if token.kind == _TOKEN_OPEN: self.pos += 1 + is_object = self._take_object_marker() multiline = self._link_is_multiline() rows = self.parse_rows(top_level=False) - return _Node(False, rows=rows, multiline=multiline) + return _Node(False, rows=rows, multiline=multiline, is_object=is_object) raise ReadableFormatError("unexpected token in readable notation") + def _take_object_marker(self) -> bool: + """Consume the ``o:`` marker if the link that just opened carries one, + which is how the single-line form says "this link is a dict, not a list".""" + if self.pos >= len(self.tokens): + return False + token = self.tokens[self.pos] + is_marker = ( + token.kind == _TOKEN_REF and not token.quoted and token.value == f"{OBJECT_MARKER}:" + ) + if is_marker: + self.pos += 1 + return is_marker + def _link_is_multiline(self) -> bool: """Whether the link that just opened spans more than one line, which is what tells an empty dict (``(\\n)``) from an empty list (``()``).""" @@ -458,10 +584,13 @@ def _link_is_multiline(self) -> bool: def _node_to_value(node: _Node) -> Any: if node.is_ref: return _ref_to_value(node.value, node.quoted) - return _rows_to_value(node.rows, node.multiline) + return _rows_to_value(node.rows, node.multiline, node.is_object) + +def _rows_to_value(rows: list[list[_Node]], multiline: bool, object_marker: bool) -> Any: + if object_marker: + return _marked_object_to_value(rows) -def _rows_to_value(rows: list[list[_Node]], multiline: bool) -> Any: if not rows: return {} if multiline else [] @@ -469,6 +598,11 @@ def _rows_to_value(rows: list[list[_Node]], multiline: bool) -> Any: if marked is not None: return marked[0] + # Written on one line, a link is a list of values: a dict on one line says so + # with the ``o:`` marker, which is what keeps ``(key value)`` unambiguous. + if not multiline: + return [_node_to_value(node) for row in rows for node in row] + # ``key value`` on every line makes a dict; anything else is a list of values. is_dict = all(len(row) == 2 and row[0].is_ref for row in rows) @@ -485,6 +619,36 @@ def _rows_to_value(rows: list[list[_Node]], multiline: bool) -> Any: return items +def _marked_object_to_value(rows: list[list[_Node]]) -> dict[str, Any]: + """Build the dict a ``(o: (key value) ...)`` link describes. + + Every value in it is a pair, so anything else is a malformed document rather + than a silent list. + """ + result: dict[str, Any] = {} + + for node in (node for row in rows for node in row): + if node.is_ref or node.is_object: + raise ReadableFormatError( + f"an object marked '{OBJECT_MARKER}:' holds (key value) pairs, " + "found a value that is not a pair" + ) + if len(node.rows) != 1: + raise ReadableFormatError( + f"an object marked '{OBJECT_MARKER}:' holds (key value) pairs, " + f"found a link of {len(node.rows)} lines" + ) + row = node.rows[0] + if len(row) != 2 or not row[0].is_ref: + raise ReadableFormatError( + f"an object marked '{OBJECT_MARKER}:' holds (key value) pairs, " + f"found a link of {len(row)} values" + ) + result[row[0].value] = _node_to_value(row[1]) + + return result + + def _decode_marked_value(rows: list[list[_Node]]) -> tuple[str] | None: """Recognise ``(base64 "...")``, the individual marker for values that could not be written as text. diff --git a/python/tests/test_readable_conformance.py b/python/tests/test_readable_conformance.py index 706ee16..84af1ef 100644 --- a/python/tests/test_readable_conformance.py +++ b/python/tests/test_readable_conformance.py @@ -2,8 +2,9 @@ The cases live in ``fixtures/readable-format/cases.json`` at the repository root and are shared by the JavaScript, Python, Rust and C# suites: every -implementation has to encode the same value to exactly the same text, which is -what keeps the four outputs byte-identical. +implementation has to encode the same value to exactly the same text and to +exactly the same single line, which is what keeps the four outputs +byte-identical. """ import json @@ -13,7 +14,7 @@ import pytest -from link_notation_objects_codec import decode, encode +from link_notation_objects_codec import decode, decode_line, encode, encode_line LANGUAGE = "python" @@ -83,3 +84,27 @@ def test_decode_matches_the_shared_value(case: dict[str, Any]) -> None: expected = _build(case["value"]) decoded = decode(case["text"]) assert _same(decoded, expected), f"{decoded!r} != {expected!r}" + + +@pytest.mark.parametrize("case", ACTIVE, ids=lambda case: case["name"]) +def test_encode_line_matches_the_shared_line(case: dict[str, Any]) -> None: + assert encode_line(_build(case["value"])) == case["line"] + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: case["name"]) +def test_the_shared_line_holds_no_line_break(case: dict[str, Any]) -> None: + assert "\n" not in case["line"] and "\r" not in case["line"], case["line"] + + +@pytest.mark.parametrize("case", ACTIVE, ids=lambda case: case["name"]) +def test_decode_line_matches_the_shared_value(case: dict[str, Any]) -> None: + expected = _build(case["value"]) + decoded = decode_line(case["line"]) + assert _same(decoded, expected), f"{decoded!r} != {expected!r}" + + +@pytest.mark.parametrize("case", ACTIVE, ids=lambda case: case["name"]) +def test_decode_reads_the_shared_line_too(case: dict[str, Any]) -> None: + expected = _build(case["value"]) + decoded = decode(case["line"]) + assert _same(decoded, expected), f"{decoded!r} != {expected!r}" diff --git a/python/tests/test_single_line_format.py b/python/tests/test_single_line_format.py new file mode 100644 index 0000000..02449fb --- /dev/null +++ b/python/tests/test_single_line_format.py @@ -0,0 +1,137 @@ +"""Tests for the readable, single-line format produced by ``encode_line()`` +(issue #43). + +An append-only log wants one record per line: appending is one write, compaction +cuts at a newline, and ``grep``, ``tail -f`` and ``wc -l`` all treat a line as an +event. ``encode()`` spreads a record over many lines and ``encode_compact()`` +hides it in base64, so neither serves that reader. +""" + +import time +from typing import Any + +import pytest +from links_notation import Parser + +from link_notation_objects_codec import ( + ReadableFormatError, + decode, + decode_line, + encode, + encode_line, +) + +#: A record of the shape an append-only log actually holds. +LOG_RECORD = { + "bytes": 2827, + "complete": True, + "server": {"host": "127.0.0.1", "port": 18878}, + "models": ["claude-haiku", "claude-opus"], +} + +LOG_RECORD_LINE = ( + '(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1")' + ' (port 18878))) (models ("claude-haiku" "claude-opus")))' +) + + +def test_a_record_is_written_on_one_line() -> None: + line = encode_line(LOG_RECORD) + assert "\n" not in line and "\r" not in line, line + assert line == LOG_RECORD_LINE + + +def test_a_line_is_valid_links_notation() -> None: + Parser().parse(encode_line(LOG_RECORD)) + + +def _ids(links: Any) -> list[str]: + """Every id in a parse tree, so a mangled parse can be recognised.""" + found: list[str] = [] + for link in links: + found.append(link.id or "") + found.extend(_ids(link.values)) + return found + + +def test_the_hand_rolled_dialect_is_not_read_as_a_record() -> None: + """The dialect a downstream project invented for the same need, which the + notation does not read back -- the reason this format exists. + + This parser does not raise on it; it swallows the parentheses into the data, + which loses the record just as surely. + """ + ids = _ids(Parser().parse('((:"bytes" 2827) (:"complete" true))')) + assert any("(" in name or ")" in name for name in ids), ids + + +@pytest.mark.parametrize( + "value", + [LOG_RECORD, [], {}, [{}, []], {"empty": []}, 42, None, "text"], +) +def test_both_forms_of_the_same_value_decode_alike(value: Any) -> None: + assert decode(encode_line(value)) == decode(encode(value)) + assert decode_line(encode_line(value)) == value + + +def test_a_string_keeps_its_own_characters_on_one_line() -> None: + value = {"text": 'quote " backslash \\ ünïcödé'} + line = encode_line(value) + assert line == "(o: (text 'quote \" backslash \\ ünïcödé'))" + assert decode_line(line) == value + + +def test_a_string_holding_a_newline_still_fits_on_one_line() -> None: + value = {"readable": "still visible", "multiline": "line1\nline2"} + line = encode_line(value) + assert line == '(o: (readable "still visible") (multiline (base64 "bGluZTEKbGluZTI=")))' + assert "\n" not in line, line + assert decode_line(line) == value + + +def test_a_one_pair_dict_is_not_a_two_element_list() -> None: + assert encode_line({"a": 1}) == "(o: (a 1))" + assert encode_line(["a", 1]) == '("a" 1)' + assert decode_line("(o: (a 1))") == {"a": 1} + assert decode_line('("a" 1)') == ["a", 1] + + +def test_the_empty_key_survives_the_round_trip() -> None: + value = {"": 2} + assert encode_line(value) == '(o: ("" 2))' + assert decode_line(encode_line(value)) == value + + +def test_a_marked_object_holding_something_that_is_not_a_pair_is_rejected() -> None: + with pytest.raises(ReadableFormatError, match="pairs"): + decode_line("(o: 1 2)") + + +def test_several_lines_are_not_one_record() -> None: + with pytest.raises(ReadableFormatError): + decode_line("(o: (a 1))\n(o: (b 2))") + + +def test_a_trailing_newline_is_not_a_second_record() -> None: + assert decode_line("(o: (a 1))\n") == {"a": 1} + + +def test_a_line_starting_with_none_is_still_read_as_a_line() -> None: + assert decode("(null 1)") == [None, 1] + assert decode("(o: (a null))") == {"a": None} + # The one document both forms claim: `(None)` is the compact null this + # language writes, and stays read that way, so documents written before this + # format keep decoding. + assert decode("(None)") is None + + +def test_a_long_run_of_line_breaks_is_rejected_without_a_slowdown() -> None: + # The JavaScript sibling trimmed the framing newlines with a regular + # expression that backtracked once per newline (CodeQL js/polynomial-redos). + # Every language strips them with a linear scan instead, and still refuses + # input holding more than one line. + notation = LOG_RECORD_LINE + "\n" * 200_000 + "x" + started = time.perf_counter() + with pytest.raises(ReadableFormatError): + decode_line(notation) + assert time.perf_counter() - started < 2.0 diff --git a/rust/README.md b/rust/README.md index 82eae66..f45abf9 100644 --- a/rust/README.md +++ b/rust/README.md @@ -32,6 +32,7 @@ lino-objects-codec = "0.1" - **Object Identity**: Shared references are preserved by the compact format - **Opt-in Tracing**: Set `LINO_CODEC_DEBUG=1` to trace encoding and decoding, the same way in every language - **Readable by Default**: `encode()` writes indented, plain-text Links Notation; keys and values stay legible and diffable +- **One Record per Line**: `encode_line()` writes the same document on one line and `decode_line()` reads it back exactly, so an append-only log stays greppable, tailable and countable by `wc -l` - **UTF-8 Support**: Full Unicode string support written as text; only values that cannot be written as text (control characters) are base64-encoded, and each is marked individually - **Simple API**: Easy-to-use `encode()` and `decode()` functions @@ -72,6 +73,7 @@ The encoded document reads as: | --- | --- | | `encode(value)` | Readable, indented Links Notation (the default) | | `encode_with_indent(value, "\t")` | Same, with a custom indentation string | +| `encode_line(value)` | The same readable document on one line, for append-only logs | | `encode_compact(value)` | The previous single-line base64 form | | `encode_obfuscated(value)` | Alias of `encode_compact` | @@ -133,6 +135,27 @@ assert_eq!(encoded, "42"); Same as `encode()`, but with a custom indentation string (the default is two spaces). +#### `encode_line(value: &LinoValue) -> String` + +Encode a value to the readable form on one line, so one record is one line of an +append-only log. An object names itself with the `o` marker, which is what keeps +`(key value)` unambiguous; a bare link on one line is always an array. + +```rust +let value = LinoValue::object([("age", LinoValue::Int(30))]); +assert_eq!(encode_line(&value), "(o: (age 30))"); +``` + +#### `decode_line(notation: &str) -> Result` + +The exact inverse of `encode_line()`. Input spanning more than one line is +rejected, so two log records can never be merged into one value. + +```rust +let decoded = decode_line("(o: (age 30))").unwrap(); +assert_eq!(decoded, LinoValue::object([("age", LinoValue::Int(30))])); +``` + #### `encode_compact(value: &LinoValue) -> String` Encode a value to the single-line, base64 form used before version 0.3. @@ -148,8 +171,8 @@ Alias of `encode_compact()`, named after what the base64 form actually does to t #### `decode(notation: &str) -> Result` -Decode Links Notation format to a value. Both the readable and the compact form -are accepted. +Decode Links Notation format to a value. The readable form, the single-line form +and the compact form are all accepted. ```rust let decoded = decode("42").unwrap(); diff --git a/rust/changelog.d/20260827_090000_issue_43_single_line_format.md b/rust/changelog.d/20260827_090000_issue_43_single_line_format.md new file mode 100644 index 0000000..73faa1b --- /dev/null +++ b/rust/changelog.d/20260827_090000_issue_43_single_line_format.md @@ -0,0 +1,25 @@ +--- +bump: minor +--- + +### Added + +- `encode_line` and `decode_line`: the readable format written on one line, so + an append-only log holds one record per line. Appending is one write, + compaction cuts at a newline, and `grep`, `tail -f` and `wc -l` treat a line + as one event. The output is valid Links Notation, keeps numbers, booleans and + `null` bare so types survive the round trip, and `decode(encode_line(v))` + equals `decode(encode(v))`. See + [issue #43](https://github.com/link-foundation/lino-objects-codec/issues/43). +- `readable::OBJECT_MARKER`, the `o` link id that tells an object from an array + on one line: `(o: (bytes 2827) (complete true))` is a record, `("a" 1)` is a + two-element array, `(o:)` is the empty object and `()` the empty array. The + marker is part of the notation, so the empty key round-trips as `(o: ("" 2))`. +- The `append_only_log` example, which writes, counts, greps and reads back a + log of one record per line. + +### Fixed + +- `decode` no longer routes a readable single-line document such as `(null 1)` + to the compact (base64) reader. Only a compact document keeps that path; the + document `(null)` stays the compact null so older documents keep decoding. diff --git a/rust/examples/append_only_log.rs b/rust/examples/append_only_log.rs new file mode 100644 index 0000000..a63a3c9 --- /dev/null +++ b/rust/examples/append_only_log.rs @@ -0,0 +1,47 @@ +//! Writing an append-only log with one record per line (issue #43). +//! +//! `encode_line` keeps a record on one line, so appending is one write, a +//! compactor can cut the file at any newline, and `grep`, `tail -f` and `wc -l` +//! all treat one line as one event. `decode_line` reads a line back exactly. + +use lino_objects_codec::{decode_line, encode_line, LinoValue}; +use std::fmt::Write as _; + +fn record(phase: &str, bytes: i64, complete: bool) -> LinoValue { + LinoValue::object([ + ("phase", LinoValue::String(phase.to_string())), + ("bytes", LinoValue::Int(bytes)), + ("complete", LinoValue::Bool(complete)), + ]) +} + +fn main() { + println!("=== Append-only log, one record per line ===\n"); + + // Appending: each record becomes exactly one line of the file. + let mut log = String::new(); + for entry in [ + record("stream_start", 0, false), + record("stream_chunk", 1024, false), + record("stream_end", 2827, true), + ] { + writeln!(log, "{}", encode_line(&entry)).expect("writing to a string cannot fail"); + } + print!("{log}"); + + // Counting: one line is one event, so `wc -l` answers how many there were. + println!("\nrecords: {}", log.lines().count()); + + // Reading: a line reader hands over one record at a time. + let last = log.lines().next_back().expect("the log holds records"); + let decoded = decode_line(last).expect("a line written by encode_line reads back"); + println!("last record: {decoded:?}"); + assert_eq!(decoded, record("stream_end", 2827, true)); + + // Filtering: the text stays readable, so plain string tools still work. + let finished = log + .lines() + .filter(|line| line.contains("(complete true)")) + .count(); + println!("finished records: {finished}"); +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 120bfdb..4aec31b 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -40,10 +40,12 @@ //! |---|---| //! | [`encode`] | readable, indented plain text (default) | //! | [`encode_with_indent`] | the same, with a custom indentation string | +//! | [`encode_line`] | readable plain text on a single line, one record per line | //! | [`encode_compact`] / [`encode_obfuscated`] | the previous single-line, base64 form | //! -//! [`decode`] accepts both forms, so files written by earlier versions keep working -//! and migrate to the readable form on the next write. +//! [`decode`] accepts all of them, so files written by earlier versions keep working +//! and migrate to the readable form on the next write. [`decode_line`] is the exact +//! inverse of [`encode_line`] and reads one record at a time. use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; use links_notation::{parse_lino_to_links, LiNo}; @@ -53,7 +55,7 @@ use std::fmt; pub mod debug; pub mod readable; -pub use readable::{BASE64_MARKER, DEFAULT_INDENT}; +pub use readable::{BASE64_MARKER, DEFAULT_INDENT, OBJECT_MARKER}; /// Type identifiers used in the compact (base64) Links Notation format mod type_ids { @@ -432,6 +434,24 @@ impl ObjectCodec { readable::encode(value, indent) } + /// Encode a LinoValue to the readable, single-line Links Notation format. + /// + /// The document is written as plain text on one line, so one value is one + /// line of an append-only log: appending is a single write, `grep`, `tail -f` + /// and `wc -l` keep working, and a compactor can cut at any newline without + /// splitting a record. See [`readable`] for the exact shape. + /// + /// # Arguments + /// + /// * `value` - The value to encode + /// + /// # Returns + /// + /// A string in readable Links Notation format, containing no newline + pub fn encode_line(&mut self, value: &LinoValue) -> String { + readable::encode_line(value) + } + /// Encode a LinoValue to the compact, single-line Links Notation format. /// /// Every value is tagged with its type and every string is base64-encoded, so @@ -711,6 +731,23 @@ impl ObjectCodec { readable::decode(notation) } + /// Decode the readable, single-line Links Notation format. + /// + /// This is the exact inverse of [`ObjectCodec::encode_line`]. A document + /// spanning more than one line is rejected, so a reader that hands over one + /// record at a time cannot silently merge two records into one value. + /// + /// # Arguments + /// + /// * `notation` - One line in readable Links Notation format + /// + /// # Returns + /// + /// The reconstructed value, or an error + pub fn decode_line(&mut self, notation: &str) -> Result { + readable::decode_line(notation) + } + /// Decode the compact (base64) Links Notation format. /// /// # Arguments @@ -945,6 +982,10 @@ const COMPACT_TYPE_MARKERS: [&str; 10] = [ "dict", ]; +/// Markers a compact document writes without a payload, so `(null)` is a compact +/// null while `(null 1)` is a readable line holding two values. +const EMPTY_BODY_MARKERS: [&str; 2] = [type_ids::NULL, "None"]; + fn is_compact_notation(notation: &str) -> bool { let Some(first_line) = notation.lines().map(str::trim).find(|l| !l.is_empty()) else { return false; @@ -954,26 +995,38 @@ fn is_compact_notation(notation: &str) -> bool { return false; }; - let mut tokens = rest - .split(|c: char| c.is_whitespace() || c == '(' || c == ')') - .filter(|t| !t.is_empty()); - - let Some(mut marker) = tokens.next() else { - return false; - }; + // A compact document names the type of its value first, so a link that opens + // another link straight away is the readable form, whose links nest. + let (mut marker, mut rest) = split_token(rest.trim_start()); // Skip the `obj_N:` definition id, if present. if let Some(id) = marker.strip_suffix(':') { if !id.starts_with("obj_") { return false; } - let Some(next) = tokens.next() else { - return false; - }; - marker = next; + (marker, rest) = split_token(rest.trim_start()); } - COMPACT_TYPE_MARKERS.contains(&marker) + if !COMPACT_TYPE_MARKERS.contains(&marker) { + return false; + } + + // A compact null is the whole link: `(null)`. A link that holds more than the + // marker is a readable line whose first value happens to be null. + if EMPTY_BODY_MARKERS.contains(&marker) { + return rest.trim_start().starts_with(')'); + } + + true +} + +/// Split off the first token of a link body: the text up to the next whitespace +/// or parenthesis. A body that opens with a parenthesis has no token of its own. +fn split_token(input: &str) -> (&str, &str) { + let end = input + .find(|c: char| c.is_whitespace() || c == '(' || c == ')') + .unwrap_or(input.len()); + input.split_at(end) } // Global codec instance for convenience functions @@ -1029,6 +1082,78 @@ pub fn encode_with_indent(value: &LinoValue, indent: &str) -> String { DEFAULT_CODEC.with(|codec| codec.borrow_mut().encode_with_indent(value, indent)) } +/// Encode a value to the readable, single-line Links Notation format. +/// +/// This is a convenience function that uses a thread-local codec instance. +/// +/// The output holds no newline, so a record written with it is a line: an +/// append-only log stays greppable, tailable and countable by `wc -l`, and the +/// values in it can be read without decoding anything. +/// +/// # Arguments +/// +/// * `value` - The value to encode +/// +/// # Returns +/// +/// A string in readable Links Notation format, containing no newline +/// +/// # Example +/// +/// ```rust +/// use lino_objects_codec::{decode_line, encode_line, LinoValue}; +/// +/// let record = LinoValue::object([ +/// ("bytes", LinoValue::Int(2827)), +/// ("complete", LinoValue::Bool(true)), +/// ("phase", LinoValue::String("stream_end".to_string())), +/// ]); +/// let line = encode_line(&record); +/// assert_eq!(line, r#"(o: (bytes 2827) (complete true) (phase "stream_end"))"#); +/// assert_eq!(decode_line(&line).unwrap(), record); +/// ``` +pub fn encode_line(value: &LinoValue) -> String { + DEFAULT_CODEC.with(|codec| codec.borrow_mut().encode_line(value)) +} + +/// Decode one line of readable Links Notation back into a value. +/// +/// This is a convenience function that uses a thread-local codec instance. It is +/// the exact inverse of [`encode_line`]; input spanning more than one line is an +/// error rather than a silently merged value. +/// +/// # Arguments +/// +/// * `notation` - One line in readable Links Notation format +/// +/// # Returns +/// +/// The reconstructed value, or an error +/// +/// # Example +/// +/// ```rust +/// use lino_objects_codec::{decode_line, LinoValue}; +/// +/// let decoded = decode_line("(o: (id 1) (tags (\"a\" \"b\")))").unwrap(); +/// assert_eq!( +/// decoded, +/// LinoValue::object([ +/// ("id", LinoValue::Int(1)), +/// ( +/// "tags", +/// LinoValue::array([ +/// LinoValue::String("a".to_string()), +/// LinoValue::String("b".to_string()), +/// ]) +/// ), +/// ]) +/// ); +/// ``` +pub fn decode_line(notation: &str) -> Result { + DEFAULT_CODEC.with(|codec| codec.borrow_mut().decode_line(notation)) +} + /// Encode a value to the compact, single-line Links Notation format. /// /// Every string is base64-encoded and the whole document is written on one line. diff --git a/rust/src/readable.rs b/rust/src/readable.rs index 6b68f80..8519756 100644 --- a/rust/src/readable.rs +++ b/rust/src/readable.rs @@ -40,6 +40,34 @@ //! control characters (including newlines and tabs, which line-based tooling and //! CRLF normalisation would corrupt) are marked individually as //! `(base64 "…")` instead of encoding the whole document. +//! +//! # Single-line form +//! +//! [`encode_line`] writes the same document on one line, so one record is one +//! line and an append-only log stays greppable, tailable and countable by +//! `wc -l`. Rows can no longer be told apart by line breaks there, so an object +//! names itself with the `o` link id the notation already has, and its pairs are +//! written as their own links: +//! +//! ```text +//! (o: (type "RouterState") (server (o: (host "127.0.0.1") (port 18878))) (models ("claude-haiku" "claude-opus"))) +//! ``` +//! +//! | Value | Single-line form | +//! |------------------|-------------------------------| +//! | `Object` | `(o: (key value) …)` | +//! | empty `Object` | `(o:)` | +//! | `Array` | `(value …)` | +//! | empty `Array` | `()` | +//! | scalars | exactly as in the indented form | +//! +//! The marker is what answers the ambiguity a flat layout otherwise has: without +//! it `((key value))` reads both as the one-pair object and as the array holding +//! the two-element array, and an empty key makes it worse. With it, a bare `( )` +//! is always an array and a marked one is always an object, so every value — +//! empty key included — survives the round trip. Consequently a *hand-written* +//! one-line link such as `(a 1)` is the two-element array, not the one-pair +//! object: on one line, objects say so. use crate::debug::trace; use crate::{CodecError, LinoValue}; @@ -51,6 +79,9 @@ pub const DEFAULT_INDENT: &str = " "; /// Marker used for values that cannot be represented as plain text. pub const BASE64_MARKER: &str = "base64"; +/// Link id naming an object in the single-line form, written as `(o: …)`. +pub const OBJECT_MARKER: &str = "o"; + /// Encode a value into the readable, indented Links Notation form. pub fn encode(value: &LinoValue, indent: &str) -> String { let mut out = String::new(); @@ -58,6 +89,31 @@ pub fn encode(value: &LinoValue, indent: &str) -> String { out } +/// Encode a value into the readable, single-line Links Notation form. +/// +/// The result never contains a newline, so one value is one line of an +/// append-only log. See the module documentation for the shape. +pub fn encode_line(value: &LinoValue) -> String { + let mut out = String::new(); + write_line_value(value, &mut out); + out +} + +/// Decode the readable, single-line Links Notation form back into a value. +/// +/// This is the exact inverse of [`encode_line`]. Input spanning more than one +/// line is rejected: a line-based reader hands over one record at a time, and +/// silently accepting several would merge two records into one value. +pub fn decode_line(text: &str) -> Result { + let line = text.trim_matches(|c: char| c == '\n' || c == '\r'); + if line.contains('\n') || line.contains('\r') { + return Err(CodecError::ParseError( + "a single-line document cannot contain a line break".to_string(), + )); + } + decode(line) +} + /// Decode the readable, indented Links Notation form back into a value. pub fn decode(text: &str) -> Result { let tokens = tokenize(text)?; @@ -76,7 +132,7 @@ pub fn decode(text: &str) -> Result { return node_to_value(&rows[0][0]); } - rows_to_value(&rows, true) + rows_to_value(&rows, true, false) } // === Encoding === @@ -126,6 +182,47 @@ fn write_value(value: &LinoValue, indent: &str, level: usize, out: &mut String) } } +/// Write a value on one line. Objects name themselves with the `o` link id and +/// write each pair as its own link, so nothing depends on where lines break. +fn write_line_value(value: &LinoValue, out: &mut String) { + match value { + LinoValue::Object(pairs) => { + if pairs.is_empty() { + // `()` is the empty array, so the empty object keeps its marker. + out.push('('); + out.push_str(OBJECT_MARKER); + out.push_str(":)"); + return; + } + + out.push('('); + out.push_str(OBJECT_MARKER); + out.push(':'); + for (key, child) in pairs { + out.push_str(" ("); + out.push_str(&format_key(key)); + out.push(' '); + write_line_value(child, out); + out.push(')'); + } + out.push(')'); + } + + LinoValue::Array(items) => { + out.push('('); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push(' '); + } + write_line_value(item, out); + } + out.push(')'); + } + + scalar => out.push_str(&format_scalar(scalar)), + } +} + fn push_indent(indent: &str, level: usize, out: &mut String) { for _ in 0..level { out.push_str(indent); @@ -232,6 +329,8 @@ enum Node { Link { rows: Vec>, multiline: bool, + /// Whether the link named itself an object with the `o:` marker. + object: bool, }, } @@ -361,9 +460,14 @@ impl Cursor { } Token::Open => { self.pos += 1; + let object = self.take_object_marker(); let multiline = self.link_is_multiline(); let rows = self.parse_rows(false)?; - Ok(Node::Link { rows, multiline }) + Ok(Node::Link { + rows, + multiline, + object, + }) } Token::Close | Token::Newline => Err(CodecError::ParseError( "unexpected token in readable notation".to_string(), @@ -371,6 +475,20 @@ impl Cursor { } } + /// Consume the `o:` marker if the link that just opened carries one, which is + /// how the single-line form says "this link is an object, not an array". + fn take_object_marker(&mut self) -> bool { + let marker = format!("{}:", OBJECT_MARKER); + let is_marker = matches!( + self.tokens.get(self.pos), + Some(Token::Ref { value, quoted: false }) if *value == marker + ); + if is_marker { + self.pos += 1; + } + is_marker + } + /// Whether the link that just opened spans more than one line, which is what /// tells an empty object (`(\n)`) from an empty array (`()`). fn link_is_multiline(&self) -> bool { @@ -384,11 +502,23 @@ impl Cursor { fn node_to_value(node: &Node) -> Result { match node { Node::Ref { value, quoted } => Ok(ref_to_value(value, *quoted)), - Node::Link { rows, multiline } => rows_to_value(rows, *multiline), + Node::Link { + rows, + multiline, + object, + } => rows_to_value(rows, *multiline, *object), } } -fn rows_to_value(rows: &[Vec], multiline: bool) -> Result { +fn rows_to_value( + rows: &[Vec], + multiline: bool, + object_marker: bool, +) -> Result { + if object_marker { + return marked_object_to_value(rows); + } + if rows.is_empty() { return Ok(if multiline { LinoValue::Object(vec![]) @@ -401,6 +531,18 @@ fn rows_to_value(rows: &[Vec], multiline: bool) -> Result], multiline: bool) -> Result]) -> Result { + let mut pairs = Vec::new(); + + for node in rows.iter().flatten() { + let Node::Link { + rows: pair, + object: false, + .. + } = node + else { + return Err(CodecError::ParseError(format!( + "an object marked '{}:' holds (key value) pairs, found a value that is not a pair", + OBJECT_MARKER + ))); + }; + + let [row] = pair.as_slice() else { + return Err(CodecError::ParseError(format!( + "an object marked '{}:' holds (key value) pairs, found a link of {} lines", + OBJECT_MARKER, + pair.len() + ))); + }; + + let [Node::Ref { value: key, .. }, value] = row.as_slice() else { + return Err(CodecError::ParseError(format!( + "an object marked '{}:' holds (key value) pairs, found a link of {} values", + OBJECT_MARKER, + row.len() + ))); + }; + + pairs.push((key.clone(), node_to_value(value)?)); + } + + Ok(LinoValue::Object(pairs)) +} + /// Recognise `(base64 "…")`, the individual marker for values that could not be /// written as text. A quoted `base64` key is an ordinary object key, not a marker. fn decode_marked_value(rows: &[Vec]) -> Option> { @@ -584,6 +766,105 @@ mod tests { ); } + fn line_roundtrip(value: &LinoValue) -> LinoValue { + let text = encode_line(value); + assert!( + !text.contains('\n'), + "line form must hold no newline: {:?}", + text + ); + decode_line(&text).unwrap_or_else(|e| panic!("failed to decode {:?}: {}", text, e)) + } + + #[test] + fn line_form_marks_objects_and_leaves_arrays_bare() { + assert_eq!(encode_line(&LinoValue::Array(vec![])), "()"); + assert_eq!(encode_line(&LinoValue::Object(vec![])), "(o:)"); + assert_eq!( + encode_line(&LinoValue::object([("a", LinoValue::Int(1))])), + "(o: (a 1))" + ); + assert_eq!( + encode_line(&LinoValue::array([ + LinoValue::String("key".into()), + LinoValue::String("value".into()), + ])), + "(\"key\" \"value\")" + ); + } + + #[test] + fn line_form_tells_a_one_pair_object_from_a_two_element_array() { + let object = LinoValue::object([("key", LinoValue::String("value".into()))]); + let array = LinoValue::array([ + LinoValue::String("key".into()), + LinoValue::String("value".into()), + ]); + assert_ne!(encode_line(&object), encode_line(&array)); + assert_eq!(line_roundtrip(&object), object); + assert_eq!(line_roundtrip(&array), array); + } + + #[test] + fn line_form_roundtrips_nested_and_empty_containers() { + let value = LinoValue::object([ + ("empty_array", LinoValue::Array(vec![])), + ("empty_object", LinoValue::Object(vec![])), + ( + "records", + LinoValue::array([ + LinoValue::object([("id", LinoValue::Int(1))]), + LinoValue::object([("id", LinoValue::Int(2))]), + ]), + ), + ]); + assert_eq!( + encode_line(&value), + "(o: (empty_array ()) (empty_object (o:)) (records ((o: (id 1)) (o: (id 2)))))" + ); + assert_eq!(line_roundtrip(&value), value); + } + + #[test] + fn line_form_keeps_the_empty_key() { + let value = LinoValue::object([("", LinoValue::Int(2))]); + assert_eq!(encode_line(&value), "(o: (\"\" 2))"); + assert_eq!(line_roundtrip(&value), value); + } + + #[test] + fn line_form_keeps_strings_on_one_line() { + let value = LinoValue::object([ + ( + "quotes", + LinoValue::String("both \"kinds\" of 'quotes'".into()), + ), + ("unicode", LinoValue::String("héllo 世界 🌍".into())), + ("multiline", LinoValue::String("line1\nline2".into())), + ]); + let text = encode_line(&value); + assert!(!text.contains('\n'), "{}", text); + assert_eq!(line_roundtrip(&value), value); + } + + #[test] + fn a_line_form_object_holds_pairs_only() { + assert!(decode_line("(o: 1 2)").is_err()); + assert!(decode_line("(o: (a 1 2))").is_err()); + assert!(decode_line("(o: (a))").is_err()); + } + + #[test] + fn decode_line_rejects_a_document_of_several_lines() { + assert!(decode_line("(o: (a 1))\n(o: (a 2))").is_err()); + // A trailing line break is what a line-based reader hands over, so it is + // stripped rather than rejected. + assert_eq!( + decode_line("(o: (a 1))\n").unwrap(), + LinoValue::object([("a", LinoValue::Int(1))]) + ); + } + #[test] fn unterminated_input_is_an_error() { assert!(decode("(\n a 1\n").is_err()); diff --git a/rust/tests/readable_conformance.rs b/rust/tests/readable_conformance.rs index 768862b..992c485 100644 --- a/rust/tests/readable_conformance.rs +++ b/rust/tests/readable_conformance.rs @@ -1,12 +1,13 @@ -//! Cross-language conformance tests for the readable, indented format. +//! Cross-language conformance tests for the readable format, indented and on a +//! single line. //! //! The fixtures in `fixtures/readable-format/cases.json` are shared by the //! JavaScript, Python, Rust and C# suites. Each case is written by hand from the //! format specification, so the four implementations check each other instead of //! agreeing on a shared mistake: every language must encode `value` to exactly -//! `text` and decode `text` back to exactly `value`. +//! `text` and to exactly `line`, and decode both back to exactly `value`. -use lino_objects_codec::{decode, encode, LinoValue}; +use lino_objects_codec::{decode, decode_line, encode, encode_line, LinoValue}; use serde_json::Value as Json; /// The language id this suite answers to in a case's `skip` map. @@ -124,6 +125,10 @@ fn text(case: &Json) -> &str { case["text"].as_str().expect("every case has a text") } +fn line(case: &Json) -> &str { + case["line"].as_str().expect("every case has a line") +} + #[test] fn every_case_is_either_active_or_skipped_with_a_reason() { let cases = cases(); @@ -199,3 +204,94 @@ fn decodes_every_shared_text_back_to_the_case_value() { failures.join("\n") ); } + +#[test] +fn encodes_every_case_to_the_shared_line() { + let mut failures = Vec::new(); + for case in cases() { + if is_skipped(&case) { + continue; + } + let encoded = encode_line(&build(&case["value"])); + if encoded != line(&case) { + failures.push(format!( + "{}: expected {:?}, got {:?}", + name(&case), + line(&case), + encoded + )); + } + } + assert!( + failures.is_empty(), + "single-line encoding mismatches:\n{}", + failures.join("\n") + ); +} + +#[test] +fn decodes_every_shared_line_back_to_the_case_value() { + let mut failures = Vec::new(); + for case in cases() { + if is_skipped(&case) { + continue; + } + let expected = build(&case["value"]); + match decode_line(line(&case)) { + Ok(decoded) if same(&decoded, &expected) => {} + Ok(decoded) => failures.push(format!( + "{}: expected {:?}, got {:?}", + name(&case), + expected, + decoded + )), + Err(error) => failures.push(format!("{}: {error}", name(&case))), + } + } + assert!( + failures.is_empty(), + "single-line decoding mismatches:\n{}", + failures.join("\n") + ); +} + +/// A log record is one line, so no case may spread over two of them. +#[test] +fn no_shared_line_contains_a_line_break() { + for case in cases() { + let line = line(&case); + assert!( + !line.contains('\n') && !line.contains('\r'), + "case {} has a line break in its single-line form: {line:?}", + name(&case) + ); + } +} + +/// `decode` reads both forms, so a log reader needs no flag saying which one it +/// holds. +#[test] +fn the_plain_decoder_reads_every_shared_line() { + let mut failures = Vec::new(); + for case in cases() { + if is_skipped(&case) { + continue; + } + let expected = build(&case["value"]); + match decode(line(&case)) { + Ok(decoded) if same(&decoded, &expected) => {} + Ok(decoded) => failures.push(format!( + "{}: expected {:?}, got {:?}", + name(&case), + expected, + decoded + )), + Err(error) => failures.push(format!("{}: {error}", name(&case))), + } + } + assert!( + failures.is_empty(), + "single-line decoding mismatches through `decode`:\n{}", + failures.join("\n") + ); +} diff --git a/rust/tests/single_line_format.rs b/rust/tests/single_line_format.rs new file mode 100644 index 0000000..489bdb3 --- /dev/null +++ b/rust/tests/single_line_format.rs @@ -0,0 +1,190 @@ +//! Tests for the readable, single-line format produced by `encode_line()` +//! (issue #43). +//! +//! An append-only log wants one record per line: appending is one write, +//! compaction cuts at a newline, and `grep`, `tail -f` and `wc -l` all treat a +//! line as an event. `encode()` spreads a record over many lines and +//! `encode_compact()` hides it in base64, so neither serves that reader. + +use links_notation::parse_lino; +use lino_objects_codec::{decode, decode_line, encode, encode_line, LinoValue}; + +/// A record of the shape an append-only log actually holds. +fn log_record() -> LinoValue { + LinoValue::object([ + ("bytes", LinoValue::Int(2827)), + ("complete", LinoValue::Bool(true)), + ( + "server", + LinoValue::object([ + ("host", LinoValue::String("127.0.0.1".to_string())), + ("port", LinoValue::Int(18878)), + ]), + ), + ( + "models", + LinoValue::array([ + LinoValue::String("claude-haiku".to_string()), + LinoValue::String("claude-opus".to_string()), + ]), + ), + ]) +} + +#[test] +fn a_record_is_written_on_one_line() { + let line = encode_line(&log_record()); + assert!( + !line.contains('\n') && !line.contains('\r'), + "a record must stay on one line, got {line:?}" + ); + assert_eq!( + line, + r#"(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1") (port 18878))) (models ("claude-haiku" "claude-opus")))"# + ); +} + +#[test] +fn a_line_is_valid_links_notation() { + let line = encode_line(&log_record()); + parse_lino(&line).unwrap_or_else(|error| { + panic!("the notation's own parser rejected {line:?}: {error:?}"); + }); +} + +/// The dialect a downstream project invented for the same need, which the +/// notation's parser rejects -- the reason this format exists. +#[test] +fn the_hand_rolled_dialect_is_the_one_the_parser_rejects() { + assert!(parse_lino(r#"((:"bytes" 2827) (:"complete" true))"#).is_err()); +} + +#[test] +fn both_forms_of_the_same_value_decode_alike() { + for value in [ + log_record(), + LinoValue::Array(vec![]), + LinoValue::Object(vec![]), + LinoValue::array([LinoValue::Object(vec![]), LinoValue::Array(vec![])]), + LinoValue::object([("empty", LinoValue::Array(vec![]))]), + LinoValue::Int(42), + LinoValue::Null, + ] { + assert_eq!( + decode(&encode_line(&value)), + decode(&encode(&value)), + "the two forms disagree about {value:?}" + ); + assert_eq!(decode_line(&encode_line(&value)), Ok(value)); + } +} + +#[test] +fn a_string_keeps_its_own_characters_on_one_line() { + let value = LinoValue::object([( + "text", + LinoValue::String("quote \" backslash \\ ünïcödé".to_string()), + )]); + let line = encode_line(&value); + assert_eq!(line, r#"(o: (text 'quote " backslash \ ünïcödé'))"#); + assert_eq!(decode_line(&line), Ok(value)); +} + +/// A newline inside a string would end the record, so such a string is the one +/// thing written encoded -- individually, so the rest of the record stays +/// readable. +#[test] +fn a_string_holding_a_newline_still_fits_on_one_line() { + let value = LinoValue::object([ + ("readable", LinoValue::String("still visible".to_string())), + ("multiline", LinoValue::String("line1\nline2".to_string())), + ]); + let line = encode_line(&value); + assert_eq!( + line, + r#"(o: (readable "still visible") (multiline (base64 "bGluZTEKbGluZTI=")))"# + ); + assert!( + !line.contains('\n'), + "a record must stay on one line: {line:?}" + ); + assert_eq!(decode_line(&line), Ok(value)); +} + +/// The one ambiguity a flat layout has: is `(a 1)` a one-pair object or a +/// two-element array? On one line an object says so with the `o:` marker, so +/// both values keep their own spelling. +#[test] +fn a_one_pair_object_is_not_a_two_element_array() { + let object = LinoValue::object([("a", LinoValue::Int(1))]); + let array = LinoValue::array([LinoValue::String("a".to_string()), LinoValue::Int(1)]); + + assert_eq!(encode_line(&object), "(o: (a 1))"); + assert_eq!(encode_line(&array), r#"("a" 1)"#); + assert_eq!(decode_line("(o: (a 1))"), Ok(object)); + assert_eq!(decode_line(r#"("a" 1)"#), Ok(array)); +} + +/// Because the marker answers it, the empty key round-trips instead of being +/// rejected: `("" 2)` is a pair like any other inside a marked object. +#[test] +fn the_empty_key_survives_the_round_trip() { + let value = LinoValue::object([("", LinoValue::Int(2))]); + assert_eq!(encode_line(&value), r#"(o: ("" 2))"#); + assert_eq!(decode_line(&encode_line(&value)), Ok(value)); +} + +#[test] +fn a_marked_object_holding_something_that_is_not_a_pair_is_rejected() { + let error = decode_line("(o: 1 2)").expect_err("a marked object holds pairs only"); + assert!( + error.to_string().contains("pairs"), + "the error must say what a marked object holds, got {error}" + ); +} + +/// Reading a log means handing over one record at a time, so a decoder that +/// silently accepted two lines would merge two records into one value. +#[test] +fn several_lines_are_not_one_record() { + assert!(decode_line("(o: (a 1))\n(o: (b 2))").is_err()); +} + +/// A trailing newline is what a line reader may keep, so it is trimmed rather +/// than refused. +#[test] +fn a_trailing_newline_is_not_a_second_record() { + assert_eq!( + decode_line("(o: (a 1))\n"), + Ok(LinoValue::object([("a", LinoValue::Int(1))])) + ); +} + +/// A line whose first value is null is a readable line, not the compact null: +/// `decode` must not route it to the base64 reader. +#[test] +fn a_line_starting_with_null_is_still_read_as_a_line() { + assert_eq!( + decode("(null 1)"), + Ok(LinoValue::array([LinoValue::Null, LinoValue::Int(1)])) + ); + assert_eq!( + decode("(o: (a null))"), + Ok(LinoValue::object([("a", LinoValue::Null)])) + ); + // The one document both forms claim: `(null)` is the compact null, and stays + // read that way, so documents written before this format keep decoding. + assert_eq!(decode("(null)"), Ok(LinoValue::Null)); +} + +/// The JavaScript sibling trimmed the framing newlines with a regular +/// expression that backtracked once per newline (CodeQL js/polynomial-redos). +/// Every language strips them with a linear scan instead, and still refuses +/// input holding more than one line. +#[test] +fn a_long_run_of_line_breaks_is_rejected_without_a_slowdown() { + let notation = format!("{}{}x", encode_line(&log_record()), "\n".repeat(200_000)); + let started = std::time::Instant::now(); + assert!(decode_line(¬ation).is_err()); + assert!(started.elapsed() < std::time::Duration::from_secs(2)); +}