From 10bc8b7f5435e8597fe96bb179b45f383fbe7bdc Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 27 Aug 2026 11:41:42 +0000 Subject: [PATCH 01/12] Initial commit with task details Adding .gitkeep for PR creation (default mode). This file will be removed when the task is complete. Issue: https://github.com/link-foundation/lino-objects-codec/issues/43 --- .gitkeep | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitkeep b/.gitkeep index ac37274..29a8cf9 100644 --- a/.gitkeep +++ b/.gitkeep @@ -1,4 +1,5 @@ # .gitkeep file auto-generated at 2026-05-10T19:22:27.543Z for PR creation at branch issue-35-03946ff48852 for issue https://github.com/link-foundation/lino-objects-codec/issues/35 # Updated: 2026-08-20T05:25:16.696Z # Updated: 2026-08-20T06:10:07.182Z -# Updated: 2026-08-20T07:45:09.136Z \ No newline at end of file +# Updated: 2026-08-20T07:45:09.136Z +# Updated: 2026-08-27T11:41:42.189Z \ No newline at end of file From bad9893e24defff2289225a6ca73d93e21c282a6 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 27 Aug 2026 11:52:39 +0000 Subject: [PATCH 02/12] feat(rust): add readable single-line encode_line/decode_line The readable format needed a layout where one record is one line: an append-only log is appended a record at a time and compacted by cutting at a newline, and grep, tail -f and wc -l all treat a line as an event. Until now the only single-line form was the base64 one, which is unreadable. On one line a link cannot say what it is by how its rows break, so an object names itself with the 'o' link id the notation already has and writes its pairs as their own links: (o: (bytes 2827) (complete true)). A bare link is then always an array, which keeps () an empty array as in the indented form, and every value -- the empty key included -- round trips. Compact-format sniffing is tightened accordingly: a link that opens another link straight away is the readable form, and the compact null is only the whole link (null), so a line such as (null 1) is read as the array it is. --- .../issue-43/line-format-probe/Cargo.lock | 81 +++++ .../issue-43/line-format-probe/Cargo.toml | 9 + .../issue-43/line-format-probe/src/main.rs | 33 ++ rust/src/lib.rs | 151 +++++++++- rust/src/readable.rs | 282 +++++++++++++++++- 5 files changed, 537 insertions(+), 19 deletions(-) create mode 100644 experiments/issue-43/line-format-probe/Cargo.lock create mode 100644 experiments/issue-43/line-format-probe/Cargo.toml create mode 100644 experiments/issue-43/line-format-probe/src/main.rs diff --git a/experiments/issue-43/line-format-probe/Cargo.lock b/experiments/issue-43/line-format-probe/Cargo.lock new file mode 100644 index 0000000..6799abd --- /dev/null +++ b/experiments/issue-43/line-format-probe/Cargo.lock @@ -0,0 +1,81 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "line-format-probe" +version = "0.1.0" +dependencies = [ + "links-notation", +] + +[[package]] +name = "links-notation" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6e5f36d99612ea82da43dbd5efb37b0b4e9c4b8197228e073b60ef5a0e78f2" +dependencies = [ + "links-notation-macro", + "nom", +] + +[[package]] +name = "links-notation-macro" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30ea96250240a92d69d45579dbd199a713e7a79acfa53033d24628dad23cff5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" diff --git a/experiments/issue-43/line-format-probe/Cargo.toml b/experiments/issue-43/line-format-probe/Cargo.toml new file mode 100644 index 0000000..e5a541f --- /dev/null +++ b/experiments/issue-43/line-format-probe/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "line-format-probe" +version = "0.1.0" +edition = "2021" + +# Probe for issue #43: which single-line spellings does the notation's own +# parser accept, and what shape do they parse into? +[dependencies] +links-notation = "0.14.0" diff --git a/experiments/issue-43/line-format-probe/src/main.rs b/experiments/issue-43/line-format-probe/src/main.rs new file mode 100644 index 0000000..4bed160 --- /dev/null +++ b/experiments/issue-43/line-format-probe/src/main.rs @@ -0,0 +1,33 @@ +use links_notation::parse_lino; + +fn main() { + let candidates = [ + // What the downstream project invented (expected: rejected). + "((:\"bytes\" 2827) (:\"complete\" true))", + // Bare groups: arrays. + "()", + "(1 2)", + "(\"key\" \"value\")", + "((1 2) (3))", + // Object marker written as a link id, which the notation already has. + "(o:)", + "(o: (a 1))", + "(o: (a 1) (b 2))", + "(o: (\"two words\" 1) (\"\" 2) (\"base64\" 3))", + "(o: (a (o: (b (o: (c (1)))))))", + "((o: (id \"1\")) (o: (id \"2\")))", + "(o: (multiline (base64 \"bGluZTEKbGluZTI=\")))", + "(o: (s 'he said \"hello\"') (t \"both \"\"kinds\"\" of 'quotes'\"))", + // Spellings we do not emit, recorded for completeness. + "(o : (a 1))", + "(a: 1 2)", + "(a:)", + ]; + + for candidate in candidates { + match parse_lino(candidate) { + Ok(parsed) => println!("OK {candidate}\n {parsed:?}"), + Err(error) => println!("ERR {candidate}\n {error}"), + } + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 120bfdb..0a52017 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -40,10 +40,12 @@ //! |---|---| //! | [`encode`] | readable, indented plain text (default) | //! | [`encode_with_indent`] | the same, with a custom indentation string | +//! | [`encode_line`] | readable plain text on a single line, one record per line | //! | [`encode_compact`] / [`encode_obfuscated`] | the previous single-line, base64 form | //! -//! [`decode`] accepts both forms, so files written by earlier versions keep working -//! and migrate to the readable form on the next write. +//! [`decode`] accepts all of them, so files written by earlier versions keep working +//! and migrate to the readable form on the next write. [`decode_line`] is the exact +//! inverse of [`encode_line`] and reads one record at a time. use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; use links_notation::{parse_lino_to_links, LiNo}; @@ -53,7 +55,7 @@ use std::fmt; pub mod debug; pub mod readable; -pub use readable::{BASE64_MARKER, DEFAULT_INDENT}; +pub use readable::{BASE64_MARKER, DEFAULT_INDENT, OBJECT_MARKER}; /// Type identifiers used in the compact (base64) Links Notation format mod type_ids { @@ -432,6 +434,24 @@ impl ObjectCodec { readable::encode(value, indent) } + /// Encode a LinoValue to the readable, single-line Links Notation format. + /// + /// The document is written as plain text on one line, so one value is one + /// line of an append-only log: appending is a single write, `grep`, `tail -f` + /// and `wc -l` keep working, and a compactor can cut at any newline without + /// splitting a record. See [`readable`] for the exact shape. + /// + /// # Arguments + /// + /// * `value` - The value to encode + /// + /// # Returns + /// + /// A string in readable Links Notation format, containing no newline + pub fn encode_line(&mut self, value: &LinoValue) -> String { + readable::encode_line(value) + } + /// Encode a LinoValue to the compact, single-line Links Notation format. /// /// Every value is tagged with its type and every string is base64-encoded, so @@ -711,6 +731,23 @@ impl ObjectCodec { readable::decode(notation) } + /// Decode the readable, single-line Links Notation format. + /// + /// This is the exact inverse of [`ObjectCodec::encode_line`]. A document + /// spanning more than one line is rejected, so a reader that hands over one + /// record at a time cannot silently merge two records into one value. + /// + /// # Arguments + /// + /// * `notation` - One line in readable Links Notation format + /// + /// # Returns + /// + /// The reconstructed value, or an error + pub fn decode_line(&mut self, notation: &str) -> Result { + readable::decode_line(notation) + } + /// Decode the compact (base64) Links Notation format. /// /// # Arguments @@ -954,26 +991,38 @@ fn is_compact_notation(notation: &str) -> bool { return false; }; - let mut tokens = rest - .split(|c: char| c.is_whitespace() || c == '(' || c == ')') - .filter(|t| !t.is_empty()); - - let Some(mut marker) = tokens.next() else { - return false; - }; + // A compact document names the type of its value first, so a link that opens + // another link straight away is the readable form, whose links nest. + let (mut marker, mut rest) = split_token(rest.trim_start()); // Skip the `obj_N:` definition id, if present. if let Some(id) = marker.strip_suffix(':') { if !id.starts_with("obj_") { return false; } - let Some(next) = tokens.next() else { - return false; - }; - marker = next; + (marker, rest) = split_token(rest.trim_start()); } - COMPACT_TYPE_MARKERS.contains(&marker) + if !COMPACT_TYPE_MARKERS.contains(&marker) { + return false; + } + + // The compact null is the whole link: `(null)`. A link that holds more than + // the marker is a readable line whose first value happens to be null. + if marker == type_ids::NULL { + return rest.trim_start().starts_with(')'); + } + + true +} + +/// Split off the first token of a link body: the text up to the next whitespace +/// or parenthesis. A body that opens with a parenthesis has no token of its own. +fn split_token(input: &str) -> (&str, &str) { + let end = input + .find(|c: char| c.is_whitespace() || c == '(' || c == ')') + .unwrap_or(input.len()); + input.split_at(end) } // Global codec instance for convenience functions @@ -1029,6 +1078,78 @@ pub fn encode_with_indent(value: &LinoValue, indent: &str) -> String { DEFAULT_CODEC.with(|codec| codec.borrow_mut().encode_with_indent(value, indent)) } +/// Encode a value to the readable, single-line Links Notation format. +/// +/// This is a convenience function that uses a thread-local codec instance. +/// +/// The output holds no newline, so a record written with it is a line: an +/// append-only log stays greppable, tailable and countable by `wc -l`, and the +/// values in it can be read without decoding anything. +/// +/// # Arguments +/// +/// * `value` - The value to encode +/// +/// # Returns +/// +/// A string in readable Links Notation format, containing no newline +/// +/// # Example +/// +/// ```rust +/// use lino_objects_codec::{decode_line, encode_line, LinoValue}; +/// +/// let record = LinoValue::object([ +/// ("bytes", LinoValue::Int(2827)), +/// ("complete", LinoValue::Bool(true)), +/// ("phase", LinoValue::String("stream_end".to_string())), +/// ]); +/// let line = encode_line(&record); +/// assert_eq!(line, r#"(o: (bytes 2827) (complete true) (phase "stream_end"))"#); +/// assert_eq!(decode_line(&line).unwrap(), record); +/// ``` +pub fn encode_line(value: &LinoValue) -> String { + DEFAULT_CODEC.with(|codec| codec.borrow_mut().encode_line(value)) +} + +/// Decode one line of readable Links Notation back into a value. +/// +/// This is a convenience function that uses a thread-local codec instance. It is +/// the exact inverse of [`encode_line`]; input spanning more than one line is an +/// error rather than a silently merged value. +/// +/// # Arguments +/// +/// * `notation` - One line in readable Links Notation format +/// +/// # Returns +/// +/// The reconstructed value, or an error +/// +/// # Example +/// +/// ```rust +/// use lino_objects_codec::{decode_line, LinoValue}; +/// +/// let decoded = decode_line("(o: (id 1) (tags (\"a\" \"b\")))").unwrap(); +/// assert_eq!( +/// decoded, +/// LinoValue::object([ +/// ("id", LinoValue::Int(1)), +/// ( +/// "tags", +/// LinoValue::array([ +/// LinoValue::String("a".to_string()), +/// LinoValue::String("b".to_string()), +/// ]) +/// ), +/// ]) +/// ); +/// ``` +pub fn decode_line(notation: &str) -> Result { + DEFAULT_CODEC.with(|codec| codec.borrow_mut().decode_line(notation)) +} + /// Encode a value to the compact, single-line Links Notation format. /// /// Every string is base64-encoded and the whole document is written on one line. diff --git a/rust/src/readable.rs b/rust/src/readable.rs index 6b68f80..268602c 100644 --- a/rust/src/readable.rs +++ b/rust/src/readable.rs @@ -40,6 +40,34 @@ //! control characters (including newlines and tabs, which line-based tooling and //! CRLF normalisation would corrupt) are marked individually as //! `(base64 "…")` instead of encoding the whole document. +//! +//! # Single-line form +//! +//! [`encode_line`] writes the same document on one line, so one record is one +//! line and an append-only log stays greppable, tailable and countable by +//! `wc -l`. Rows can no longer be told apart by line breaks there, so an object +//! names itself with the `o` link id the notation already has, and its pairs are +//! written as their own links: +//! +//! ```text +//! (o: (type "RouterState") (server (o: (host "127.0.0.1") (port 18878))) (models ("claude-haiku" "claude-opus"))) +//! ``` +//! +//! | Value | Single-line form | +//! |------------------|-------------------------------| +//! | `Object` | `(o: (key value) …)` | +//! | empty `Object` | `(o:)` | +//! | `Array` | `(value …)` | +//! | empty `Array` | `()` | +//! | scalars | exactly as in the indented form | +//! +//! The marker is what answers the ambiguity a flat layout otherwise has: without +//! it `((key value))` reads both as the one-pair object and as the array holding +//! the two-element array, and an empty key makes it worse. With it, a bare `( )` +//! is always an array and a marked one is always an object, so every value — +//! empty key included — survives the round trip. Consequently a *hand-written* +//! one-line link such as `(a 1)` is the two-element array, not the one-pair +//! object: on one line, objects say so. use crate::debug::trace; use crate::{CodecError, LinoValue}; @@ -51,6 +79,9 @@ pub const DEFAULT_INDENT: &str = " "; /// Marker used for values that cannot be represented as plain text. pub const BASE64_MARKER: &str = "base64"; +/// Link id naming an object in the single-line form, written as `(o: …)`. +pub const OBJECT_MARKER: &str = "o"; + /// Encode a value into the readable, indented Links Notation form. pub fn encode(value: &LinoValue, indent: &str) -> String { let mut out = String::new(); @@ -58,6 +89,31 @@ pub fn encode(value: &LinoValue, indent: &str) -> String { out } +/// Encode a value into the readable, single-line Links Notation form. +/// +/// The result never contains a newline, so one value is one line of an +/// append-only log. See the module documentation for the shape. +pub fn encode_line(value: &LinoValue) -> String { + let mut out = String::new(); + write_line_value(value, &mut out); + out +} + +/// Decode the readable, single-line Links Notation form back into a value. +/// +/// This is the exact inverse of [`encode_line`]. Input spanning more than one +/// line is rejected: a line-based reader hands over one record at a time, and +/// silently accepting several would merge two records into one value. +pub fn decode_line(text: &str) -> Result { + let line = text.trim_matches(|c: char| c == '\n' || c == '\r'); + if line.contains('\n') || line.contains('\r') { + return Err(CodecError::ParseError( + "a single-line document cannot contain a line break".to_string(), + )); + } + decode(line) +} + /// Decode the readable, indented Links Notation form back into a value. pub fn decode(text: &str) -> Result { let tokens = tokenize(text)?; @@ -76,7 +132,7 @@ pub fn decode(text: &str) -> Result { return node_to_value(&rows[0][0]); } - rows_to_value(&rows, true) + rows_to_value(&rows, true, false) } // === Encoding === @@ -126,6 +182,47 @@ fn write_value(value: &LinoValue, indent: &str, level: usize, out: &mut String) } } +/// Write a value on one line. Objects name themselves with the `o` link id and +/// write each pair as its own link, so nothing depends on where lines break. +fn write_line_value(value: &LinoValue, out: &mut String) { + match value { + LinoValue::Object(pairs) => { + if pairs.is_empty() { + // `()` is the empty array, so the empty object keeps its marker. + out.push('('); + out.push_str(OBJECT_MARKER); + out.push_str(":)"); + return; + } + + out.push('('); + out.push_str(OBJECT_MARKER); + out.push(':'); + for (key, child) in pairs { + out.push_str(" ("); + out.push_str(&format_key(key)); + out.push(' '); + write_line_value(child, out); + out.push(')'); + } + out.push(')'); + } + + LinoValue::Array(items) => { + out.push('('); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push(' '); + } + write_line_value(item, out); + } + out.push(')'); + } + + scalar => out.push_str(&format_scalar(scalar)), + } +} + fn push_indent(indent: &str, level: usize, out: &mut String) { for _ in 0..level { out.push_str(indent); @@ -232,6 +329,8 @@ enum Node { Link { rows: Vec>, multiline: bool, + /// Whether the link named itself an object with the `o:` marker. + object: bool, }, } @@ -361,9 +460,14 @@ impl Cursor { } Token::Open => { self.pos += 1; + let object = self.take_object_marker(); let multiline = self.link_is_multiline(); let rows = self.parse_rows(false)?; - Ok(Node::Link { rows, multiline }) + Ok(Node::Link { + rows, + multiline, + object, + }) } Token::Close | Token::Newline => Err(CodecError::ParseError( "unexpected token in readable notation".to_string(), @@ -371,6 +475,20 @@ impl Cursor { } } + /// Consume the `o:` marker if the link that just opened carries one, which is + /// how the single-line form says "this link is an object, not an array". + fn take_object_marker(&mut self) -> bool { + let marker = format!("{}:", OBJECT_MARKER); + let is_marker = matches!( + self.tokens.get(self.pos), + Some(Token::Ref { value, quoted: false }) if *value == marker + ); + if is_marker { + self.pos += 1; + } + is_marker + } + /// Whether the link that just opened spans more than one line, which is what /// tells an empty object (`(\n)`) from an empty array (`()`). fn link_is_multiline(&self) -> bool { @@ -384,11 +502,23 @@ impl Cursor { fn node_to_value(node: &Node) -> Result { match node { Node::Ref { value, quoted } => Ok(ref_to_value(value, *quoted)), - Node::Link { rows, multiline } => rows_to_value(rows, *multiline), + Node::Link { + rows, + multiline, + object, + } => rows_to_value(rows, *multiline, *object), } } -fn rows_to_value(rows: &[Vec], multiline: bool) -> Result { +fn rows_to_value( + rows: &[Vec], + multiline: bool, + object_marker: bool, +) -> Result { + if object_marker { + return marked_object_to_value(rows); + } + if rows.is_empty() { return Ok(if multiline { LinoValue::Object(vec![]) @@ -401,6 +531,18 @@ fn rows_to_value(rows: &[Vec], multiline: bool) -> Result], multiline: bool) -> Result]) -> Result { + let mut pairs = Vec::new(); + + for node in rows.iter().flatten() { + let Node::Link { + rows: pair, + object: false, + .. + } = node + else { + return Err(CodecError::ParseError(format!( + "an object marked '{}:' holds (key value) pairs, found a value that is not a pair", + OBJECT_MARKER + ))); + }; + + let [row] = pair.as_slice() else { + return Err(CodecError::ParseError(format!( + "an object marked '{}:' holds (key value) pairs, found a link of {} lines", + OBJECT_MARKER, + pair.len() + ))); + }; + + let [Node::Ref { value: key, .. }, value] = row.as_slice() else { + return Err(CodecError::ParseError(format!( + "an object marked '{}:' holds (key value) pairs, found a link of {} values", + OBJECT_MARKER, + row.len() + ))); + }; + + pairs.push((key.clone(), node_to_value(value)?)); + } + + Ok(LinoValue::Object(pairs)) +} + /// Recognise `(base64 "…")`, the individual marker for values that could not be /// written as text. A quoted `base64` key is an ordinary object key, not a marker. fn decode_marked_value(rows: &[Vec]) -> Option> { @@ -584,6 +766,98 @@ mod tests { ); } + fn line_roundtrip(value: &LinoValue) -> LinoValue { + let text = encode_line(value); + assert!(!text.contains('\n'), "line form must hold no newline: {:?}", text); + decode_line(&text).unwrap_or_else(|e| panic!("failed to decode {:?}: {}", text, e)) + } + + #[test] + fn line_form_marks_objects_and_leaves_arrays_bare() { + assert_eq!(encode_line(&LinoValue::Array(vec![])), "()"); + assert_eq!(encode_line(&LinoValue::Object(vec![])), "(o:)"); + assert_eq!( + encode_line(&LinoValue::object([("a", LinoValue::Int(1))])), + "(o: (a 1))" + ); + assert_eq!( + encode_line(&LinoValue::array([ + LinoValue::String("key".into()), + LinoValue::String("value".into()), + ])), + "(\"key\" \"value\")" + ); + } + + #[test] + fn line_form_tells_a_one_pair_object_from_a_two_element_array() { + let object = LinoValue::object([("key", LinoValue::String("value".into()))]); + let array = LinoValue::array([ + LinoValue::String("key".into()), + LinoValue::String("value".into()), + ]); + assert_ne!(encode_line(&object), encode_line(&array)); + assert_eq!(line_roundtrip(&object), object); + assert_eq!(line_roundtrip(&array), array); + } + + #[test] + fn line_form_roundtrips_nested_and_empty_containers() { + let value = LinoValue::object([ + ("empty_array", LinoValue::Array(vec![])), + ("empty_object", LinoValue::Object(vec![])), + ( + "records", + LinoValue::array([ + LinoValue::object([("id", LinoValue::Int(1))]), + LinoValue::object([("id", LinoValue::Int(2))]), + ]), + ), + ]); + assert_eq!( + encode_line(&value), + "(o: (empty_array ()) (empty_object (o:)) (records ((o: (id 1)) (o: (id 2)))))" + ); + assert_eq!(line_roundtrip(&value), value); + } + + #[test] + fn line_form_keeps_the_empty_key() { + let value = LinoValue::object([("", LinoValue::Int(2))]); + assert_eq!(encode_line(&value), "(o: (\"\" 2))"); + assert_eq!(line_roundtrip(&value), value); + } + + #[test] + fn line_form_keeps_strings_on_one_line() { + let value = LinoValue::object([ + ("quotes", LinoValue::String("both \"kinds\" of 'quotes'".into())), + ("unicode", LinoValue::String("héllo 世界 🌍".into())), + ("multiline", LinoValue::String("line1\nline2".into())), + ]); + let text = encode_line(&value); + assert!(!text.contains('\n'), "{}", text); + assert_eq!(line_roundtrip(&value), value); + } + + #[test] + fn a_line_form_object_holds_pairs_only() { + assert!(decode_line("(o: 1 2)").is_err()); + assert!(decode_line("(o: (a 1 2))").is_err()); + assert!(decode_line("(o: (a))").is_err()); + } + + #[test] + fn decode_line_rejects_a_document_of_several_lines() { + assert!(decode_line("(o: (a 1))\n(o: (a 2))").is_err()); + // A trailing line break is what a line-based reader hands over, so it is + // stripped rather than rejected. + assert_eq!( + decode_line("(o: (a 1))\n").unwrap(), + LinoValue::object([("a", LinoValue::Int(1))]) + ); + } + #[test] fn unterminated_input_is_an_error() { assert!(decode("(\n a 1\n").is_err()); From 55ef616a4a8a6705a97b9d01890a740e69ceb69f Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 27 Aug 2026 11:55:28 +0000 Subject: [PATCH 03/12] test(fixtures): pin the single-line form in the shared cases --- experiments/issue-43/build_line_fixtures.py | 91 +++++++++ fixtures/readable-format/cases.json | 209 ++++++++++++++++---- rust/tests/readable_conformance.rs | 102 +++++++++- 3 files changed, 358 insertions(+), 44 deletions(-) create mode 100644 experiments/issue-43/build_line_fixtures.py diff --git a/experiments/issue-43/build_line_fixtures.py b/experiments/issue-43/build_line_fixtures.py new file mode 100644 index 0000000..75dea68 --- /dev/null +++ b/experiments/issue-43/build_line_fixtures.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Fill in the `line` field of the shared readable-format fixtures. + +Written directly from the single-line specification in `rust/src/readable.rs`, +independently of any implementation, so that the four language suites are +checked against the format rather than against one of them. + +Usage: python3 experiments/issue-43/build_line_fixtures.py [--write] +""" +import base64 +import json +import pathlib +import sys + +FIXTURES = pathlib.Path(__file__).resolve().parents[2] / "fixtures/readable-format/cases.json" +BASE64_MARKER = "base64" +OBJECT_MARKER = "o" +KEY_NEEDS_QUOTES = set(" \t\n\r()'\":`") + + +def needs_encoding(text): + return any(ord(c) <= 0x1F or 0x7F <= ord(c) <= 0x9F for c in text) + + +def quote(text): + if '"' not in text: + return f'"{text}"' + if "'" not in text: + return f"'{text}'" + return '"' + text.replace('"', '""') + '"' + + +def format_string(text): + if needs_encoding(text): + payload = base64.b64encode(text.encode("utf-8")).decode("ascii") + return f"({BASE64_MARKER} {quote(payload)})" + return quote(text) + + +def format_key(key): + plain = ( + key != "" + and key != BASE64_MARKER + and not needs_encoding(key) + and not any(c in KEY_NEEDS_QUOTES or c.isspace() for c in key) + ) + return key if plain else format_string(key) + + +def format_float(value): + if isinstance(value, str): + return value # "NaN", "Infinity", "-Infinity" + text = repr(float(value)) + return text + + +def line_of(tagged): + (tag, payload), = tagged.items() + if tag == "null": + return "null" + if tag == "bool": + return "true" if payload else "false" + if tag == "int": + return str(payload) + if tag == "float": + return format_float(payload) + if tag == "str": + return format_string(payload) + if tag == "array": + return "(" + " ".join(line_of(item) for item in payload) + ")" + if tag == "object": + if not payload: + return f"({OBJECT_MARKER}:)" + pairs = " ".join(f"({format_key(k)} {line_of(v)})" for k, v in payload) + return f"({OBJECT_MARKER}: {pairs})" + raise SystemExit(f"unknown tag {tag}") + + +def main(): + document = json.loads(FIXTURES.read_text()) + for case in document["cases"]: + case["line"] = line_of(case["value"]) + print(f"{case['name']}: {case['line']}") + + if "--write" in sys.argv: + FIXTURES.write_text(json.dumps(document, indent=2, ensure_ascii=False) + "\n") + print(f"\nwrote {FIXTURES}") + + +if __name__ == "__main__": + main() diff --git a/fixtures/readable-format/cases.json b/fixtures/readable-format/cases.json index e2b3f1e..c6465fe 100644 --- a/fixtures/readable-format/cases.json +++ b/fixtures/readable-format/cases.json @@ -1,5 +1,5 @@ { - "description": "Shared conformance fixtures for the readable, indented Links Notation format. Every implementation must encode `value` to exactly `text` and decode `text` back to exactly `value`, so the four languages produce byte-identical documents.", + "description": "Shared conformance fixtures for the readable Links Notation format. Every implementation must encode `value` to exactly `text` in the indented form and to exactly `line` in the single-line form, and decode both back to exactly `value`, so the four languages produce byte-identical documents.", "valueEncoding": "A value is a single-key object naming its type: {\"null\": true}, {\"bool\": …}, {\"int\": …}, {\"float\": … | \"NaN\" | \"Infinity\" | \"-Infinity\"}, {\"str\": …}, {\"array\": [value, …]} or {\"object\": [[key, value], …]}. Object pairs are a list, not a map, because key order is part of the document.", "skip": "`skip` maps a language id (js, python, rust, csharp) to the reason its value model cannot represent the case. A suite skips exactly the cases naming it.", "cases": [ @@ -8,49 +8,56 @@ "value": { "null": true }, - "text": "null" + "text": "null", + "line": "null" }, { "name": "bool_true", "value": { "bool": true }, - "text": "true" + "text": "true", + "line": "true" }, { "name": "bool_false", "value": { "bool": false }, - "text": "false" + "text": "false", + "line": "false" }, { "name": "int_positive", "value": { "int": 42 }, - "text": "42" + "text": "42", + "line": "42" }, { "name": "int_negative", "value": { "int": -7 }, - "text": "-7" + "text": "-7", + "line": "-7" }, { "name": "int_zero", "value": { "int": 0 }, - "text": "0" + "text": "0", + "line": "0" }, { "name": "float_fraction", "value": { "float": 3.5 }, - "text": "3.5" + "text": "3.5", + "line": "3.5" }, { "name": "float_whole", @@ -60,133 +67,152 @@ "text": "2.0", "skip": { "js": "JavaScript has one number type, so 2.0 and 2 are the same value and the trailing '.0' cannot be recovered when encoding" - } + }, + "line": "2.0" }, { "name": "float_negative", "value": { "float": -0.5 }, - "text": "-0.5" + "text": "-0.5", + "line": "-0.5" }, { "name": "float_nan", "value": { "float": "NaN" }, - "text": "NaN" + "text": "NaN", + "line": "NaN" }, { "name": "float_infinity", "value": { "float": "Infinity" }, - "text": "Infinity" + "text": "Infinity", + "line": "Infinity" }, { "name": "float_negative_infinity", "value": { "float": "-Infinity" }, - "text": "-Infinity" + "text": "-Infinity", + "line": "-Infinity" }, { "name": "string_plain", "value": { "str": "root" }, - "text": "\"root\"" + "text": "\"root\"", + "line": "\"root\"" }, { "name": "string_empty", "value": { "str": "" }, - "text": "\"\"" + "text": "\"\"", + "line": "\"\"" }, { "name": "string_with_spaces", "value": { "str": "with spaces" }, - "text": "\"with spaces\"" + "text": "\"with spaces\"", + "line": "\"with spaces\"" }, { "name": "string_apostrophe", "value": { "str": "it's" }, - "text": "\"it's\"" + "text": "\"it's\"", + "line": "\"it's\"" }, { "name": "string_double_quote", "value": { "str": "he said \"hello\"" }, - "text": "'he said \"hello\"'" + "text": "'he said \"hello\"'", + "line": "'he said \"hello\"'" }, { "name": "string_both_quote_kinds", "value": { "str": "both \"kinds\" of 'quotes'" }, - "text": "\"both \"\"kinds\"\" of 'quotes'\"" + "text": "\"both \"\"kinds\"\" of 'quotes'\"", + "line": "\"both \"\"kinds\"\" of 'quotes'\"" }, { "name": "string_unicode", "value": { "str": "unicode: 你好世界 🌍" }, - "text": "\"unicode: 你好世界 🌍\"" + "text": "\"unicode: 你好世界 🌍\"", + "line": "\"unicode: 你好世界 🌍\"" }, { "name": "string_parens_and_colon", "value": { "str": "parens (and) colons: yes" }, - "text": "\"parens (and) colons: yes\"" + "text": "\"parens (and) colons: yes\"", + "line": "\"parens (and) colons: yes\"" }, { "name": "string_numeric", "value": { "str": "18878" }, - "text": "\"18878\"" + "text": "\"18878\"", + "line": "\"18878\"" }, { "name": "string_boolean", "value": { "str": "true" }, - "text": "\"true\"" + "text": "\"true\"", + "line": "\"true\"" }, { "name": "string_with_newline", "value": { "str": "line1\nline2" }, - "text": "(base64 \"bGluZTEKbGluZTI=\")" + "text": "(base64 \"bGluZTEKbGluZTI=\")", + "line": "(base64 \"bGluZTEKbGluZTI=\")" }, { "name": "string_with_tab", "value": { "str": "a\tb" }, - "text": "(base64 \"YQli\")" + "text": "(base64 \"YQli\")", + "line": "(base64 \"YQli\")" }, { "name": "empty_array", "value": { "array": [] }, - "text": "()" + "text": "()", + "line": "()" }, { "name": "empty_object", "value": { "object": [] }, - "text": "(\n)" + "text": "(\n)", + "line": "(o:)" }, { "name": "array_of_scalars", @@ -206,7 +232,8 @@ } ] }, - "text": "(\n 1\n \"two\"\n true\n null\n)" + "text": "(\n 1\n \"two\"\n true\n null\n)", + "line": "(1 \"two\" true null)" }, { "name": "object_of_scalars", @@ -226,7 +253,8 @@ ] ] }, - "text": "(\n name \"Alice\"\n age 30\n)" + "text": "(\n name \"Alice\"\n age 30\n)", + "line": "(o: (name \"Alice\") (age 30))" }, { "name": "nested_empty_containers", @@ -246,7 +274,8 @@ ] ] }, - "text": "(\n empty_array ()\n empty_object (\n )\n)" + "text": "(\n empty_array ()\n empty_object (\n )\n)", + "line": "(o: (empty_array ()) (empty_object (o:)))" }, { "name": "array_of_objects_keeps_record_boundaries", @@ -286,7 +315,8 @@ } ] }, - "text": "(\n (\n id \"1\"\n label \"one\"\n )\n (\n id \"2\"\n label \"two\"\n )\n)" + "text": "(\n (\n id \"1\"\n label \"one\"\n )\n (\n id \"2\"\n label \"two\"\n )\n)", + "line": "((o: (id \"1\") (label \"one\")) (o: (id \"2\") (label \"two\")))" }, { "name": "array_of_arrays", @@ -311,7 +341,95 @@ } ] }, - "text": "(\n (\n 1\n 2\n )\n (\n 3\n )\n)" + "text": "(\n (\n 1\n 2\n )\n (\n 3\n )\n)", + "line": "((1 2) (3))" + }, + { + "name": "array_of_pairs_is_not_an_object", + "value": { + "array": [ + { + "array": [ + { + "str": "a" + }, + { + "int": 1 + } + ] + }, + { + "array": [ + { + "str": "b" + }, + { + "int": 2 + } + ] + } + ] + }, + "text": "(\n (\n \"a\"\n 1\n )\n (\n \"b\"\n 2\n )\n)", + "line": "((\"a\" 1) (\"b\" 2))" + }, + { + "name": "array_holding_one_object", + "value": { + "array": [ + { + "object": [ + [ + "a", + { + "int": 1 + } + ] + ] + } + ] + }, + "text": "(\n (\n a 1\n )\n)", + "line": "((o: (a 1)))" + }, + { + "name": "array_holding_an_empty_object", + "value": { + "array": [ + { + "object": [] + } + ] + }, + "text": "(\n (\n )\n)", + "line": "((o:))" + }, + { + "name": "log_record_of_one_line", + "value": { + "object": [ + [ + "bytes", + { + "int": 2827 + } + ], + [ + "complete", + { + "bool": true + } + ], + [ + "phase", + { + "str": "stream_end" + } + ] + ] + }, + "text": "(\n bytes 2827\n complete true\n phase \"stream_end\"\n)", + "line": "(o: (bytes 2827) (complete true) (phase \"stream_end\"))" }, { "name": "single_pair_object_is_not_a_two_element_array", @@ -325,7 +443,8 @@ ] ] }, - "text": "(\n key \"value\"\n)" + "text": "(\n key \"value\"\n)", + "line": "(o: (key \"value\"))" }, { "name": "two_element_array_is_not_a_single_pair_object", @@ -339,7 +458,8 @@ } ] }, - "text": "(\n \"key\"\n \"value\"\n)" + "text": "(\n \"key\"\n \"value\"\n)", + "line": "(\"key\" \"value\")" }, { "name": "keys_that_need_quoting", @@ -377,7 +497,8 @@ ] ] }, - "text": "(\n \"two words\" 1\n \"\" 2\n \"base64\" 3\n \"with:colon\" 4\n 'with\"quote' 5\n)" + "text": "(\n \"two words\" 1\n \"\" 2\n \"base64\" 3\n \"with:colon\" 4\n 'with\"quote' 5\n)", + "line": "(o: (\"two words\" 1) (\"\" 2) (\"base64\" 3) (\"with:colon\" 4) ('with\"quote' 5))" }, { "name": "base64_key_with_plain_value_is_not_a_marker", @@ -391,7 +512,8 @@ ] ] }, - "text": "(\n \"base64\" \"plain text\"\n)" + "text": "(\n \"base64\" \"plain text\"\n)", + "line": "(o: (\"base64\" \"plain text\"))" }, { "name": "documented_router_state", @@ -437,7 +559,8 @@ ] ] }, - "text": "(\n type \"RouterState\"\n server (\n host \"127.0.0.1\"\n port 18878\n )\n models (\n \"claude-haiku\"\n \"claude-opus\"\n )\n)" + "text": "(\n type \"RouterState\"\n server (\n host \"127.0.0.1\"\n port 18878\n )\n models (\n \"claude-haiku\"\n \"claude-opus\"\n )\n)", + "line": "(o: (type \"RouterState\") (server (o: (host \"127.0.0.1\") (port 18878))) (models (\"claude-haiku\" \"claude-opus\")))" }, { "name": "mixed_types_in_one_object", @@ -496,7 +619,8 @@ "text": "(\n int -7\n float 3.5\n whole_float 2.0\n yes true\n no false\n nothing null\n numeric_string \"18878\"\n boolean_string \"true\"\n)", "skip": { "js": "JavaScript has one number type, so 2.0 and 2 are the same value and the trailing '.0' cannot be recovered when encoding" - } + }, + "line": "(o: (int -7) (float 3.5) (whole_float 2.0) (yes true) (no false) (nothing null) (numeric_string \"18878\") (boolean_string \"true\"))" }, { "name": "only_unwritable_values_are_marked", @@ -522,7 +646,8 @@ ] ] }, - "text": "(\n readable \"still visible\"\n multiline (base64 \"bGluZTEKbGluZTI=\")\n tabbed (base64 \"YQli\")\n)" + "text": "(\n readable \"still visible\"\n multiline (base64 \"bGluZTEKbGluZTI=\")\n tabbed (base64 \"YQli\")\n)", + "line": "(o: (readable \"still visible\") (multiline (base64 \"bGluZTEKbGluZTI=\")) (tabbed (base64 \"YQli\")))" }, { "name": "deeply_nested_objects", @@ -554,7 +679,9 @@ ] ] }, - "text": "(\n a (\n b (\n c (\n 1\n )\n )\n )\n)" + "text": "(\n a (\n b (\n c (\n 1\n )\n )\n )\n)", + "line": "(o: (a (o: (b (o: (c (1)))))))" } - ] + ], + "lineEncoding": "`line` is the single-line form: the same document written without any line break, one record per line. Line breaks no longer separate rows there, so an object names itself with the `o` link id (`(o: (key value) ...)`, `(o:)` when empty) and a bare link is always an array." } diff --git a/rust/tests/readable_conformance.rs b/rust/tests/readable_conformance.rs index 768862b..992c485 100644 --- a/rust/tests/readable_conformance.rs +++ b/rust/tests/readable_conformance.rs @@ -1,12 +1,13 @@ -//! Cross-language conformance tests for the readable, indented format. +//! Cross-language conformance tests for the readable format, indented and on a +//! single line. //! //! The fixtures in `fixtures/readable-format/cases.json` are shared by the //! JavaScript, Python, Rust and C# suites. Each case is written by hand from the //! format specification, so the four implementations check each other instead of //! agreeing on a shared mistake: every language must encode `value` to exactly -//! `text` and decode `text` back to exactly `value`. +//! `text` and to exactly `line`, and decode both back to exactly `value`. -use lino_objects_codec::{decode, encode, LinoValue}; +use lino_objects_codec::{decode, decode_line, encode, encode_line, LinoValue}; use serde_json::Value as Json; /// The language id this suite answers to in a case's `skip` map. @@ -124,6 +125,10 @@ fn text(case: &Json) -> &str { case["text"].as_str().expect("every case has a text") } +fn line(case: &Json) -> &str { + case["line"].as_str().expect("every case has a line") +} + #[test] fn every_case_is_either_active_or_skipped_with_a_reason() { let cases = cases(); @@ -199,3 +204,94 @@ fn decodes_every_shared_text_back_to_the_case_value() { failures.join("\n") ); } + +#[test] +fn encodes_every_case_to_the_shared_line() { + let mut failures = Vec::new(); + for case in cases() { + if is_skipped(&case) { + continue; + } + let encoded = encode_line(&build(&case["value"])); + if encoded != line(&case) { + failures.push(format!( + "{}: expected {:?}, got {:?}", + name(&case), + line(&case), + encoded + )); + } + } + assert!( + failures.is_empty(), + "single-line encoding mismatches:\n{}", + failures.join("\n") + ); +} + +#[test] +fn decodes_every_shared_line_back_to_the_case_value() { + let mut failures = Vec::new(); + for case in cases() { + if is_skipped(&case) { + continue; + } + let expected = build(&case["value"]); + match decode_line(line(&case)) { + Ok(decoded) if same(&decoded, &expected) => {} + Ok(decoded) => failures.push(format!( + "{}: expected {:?}, got {:?}", + name(&case), + expected, + decoded + )), + Err(error) => failures.push(format!("{}: {error}", name(&case))), + } + } + assert!( + failures.is_empty(), + "single-line decoding mismatches:\n{}", + failures.join("\n") + ); +} + +/// A log record is one line, so no case may spread over two of them. +#[test] +fn no_shared_line_contains_a_line_break() { + for case in cases() { + let line = line(&case); + assert!( + !line.contains('\n') && !line.contains('\r'), + "case {} has a line break in its single-line form: {line:?}", + name(&case) + ); + } +} + +/// `decode` reads both forms, so a log reader needs no flag saying which one it +/// holds. +#[test] +fn the_plain_decoder_reads_every_shared_line() { + let mut failures = Vec::new(); + for case in cases() { + if is_skipped(&case) { + continue; + } + let expected = build(&case["value"]); + match decode(line(&case)) { + Ok(decoded) if same(&decoded, &expected) => {} + Ok(decoded) => failures.push(format!( + "{}: expected {:?}, got {:?}", + name(&case), + expected, + decoded + )), + Err(error) => failures.push(format!("{}: {error}", name(&case))), + } + } + assert!( + failures.is_empty(), + "single-line decoding mismatches through `decode`:\n{}", + failures.join("\n") + ); +} From 9b8615ec0bf1706c5d6e80200aeaa033f0b7b52b Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 27 Aug 2026 11:57:42 +0000 Subject: [PATCH 04/12] test(rust): cover the single-line form end to end --- rust/src/lib.rs | 10 +- rust/src/readable.rs | 11 +- rust/tests/single_line_format.rs | 178 +++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 rust/tests/single_line_format.rs diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 0a52017..4aec31b 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -982,6 +982,10 @@ const COMPACT_TYPE_MARKERS: [&str; 10] = [ "dict", ]; +/// Markers a compact document writes without a payload, so `(null)` is a compact +/// null while `(null 1)` is a readable line holding two values. +const EMPTY_BODY_MARKERS: [&str; 2] = [type_ids::NULL, "None"]; + fn is_compact_notation(notation: &str) -> bool { let Some(first_line) = notation.lines().map(str::trim).find(|l| !l.is_empty()) else { return false; @@ -1007,9 +1011,9 @@ fn is_compact_notation(notation: &str) -> bool { return false; } - // The compact null is the whole link: `(null)`. A link that holds more than - // the marker is a readable line whose first value happens to be null. - if marker == type_ids::NULL { + // A compact null is the whole link: `(null)`. A link that holds more than the + // marker is a readable line whose first value happens to be null. + if EMPTY_BODY_MARKERS.contains(&marker) { return rest.trim_start().starts_with(')'); } diff --git a/rust/src/readable.rs b/rust/src/readable.rs index 268602c..8519756 100644 --- a/rust/src/readable.rs +++ b/rust/src/readable.rs @@ -768,7 +768,11 @@ mod tests { fn line_roundtrip(value: &LinoValue) -> LinoValue { let text = encode_line(value); - assert!(!text.contains('\n'), "line form must hold no newline: {:?}", text); + assert!( + !text.contains('\n'), + "line form must hold no newline: {:?}", + text + ); decode_line(&text).unwrap_or_else(|e| panic!("failed to decode {:?}: {}", text, e)) } @@ -831,7 +835,10 @@ mod tests { #[test] fn line_form_keeps_strings_on_one_line() { let value = LinoValue::object([ - ("quotes", LinoValue::String("both \"kinds\" of 'quotes'".into())), + ( + "quotes", + LinoValue::String("both \"kinds\" of 'quotes'".into()), + ), ("unicode", LinoValue::String("héllo 世界 🌍".into())), ("multiline", LinoValue::String("line1\nline2".into())), ]); diff --git a/rust/tests/single_line_format.rs b/rust/tests/single_line_format.rs new file mode 100644 index 0000000..9ab2e83 --- /dev/null +++ b/rust/tests/single_line_format.rs @@ -0,0 +1,178 @@ +//! Tests for the readable, single-line format produced by `encode_line()` +//! (issue #43). +//! +//! An append-only log wants one record per line: appending is one write, +//! compaction cuts at a newline, and `grep`, `tail -f` and `wc -l` all treat a +//! line as an event. `encode()` spreads a record over many lines and +//! `encode_compact()` hides it in base64, so neither serves that reader. + +use links_notation::parse_lino; +use lino_objects_codec::{decode, decode_line, encode, encode_line, LinoValue}; + +/// A record of the shape an append-only log actually holds. +fn log_record() -> LinoValue { + LinoValue::object([ + ("bytes", LinoValue::Int(2827)), + ("complete", LinoValue::Bool(true)), + ( + "server", + LinoValue::object([ + ("host", LinoValue::String("127.0.0.1".to_string())), + ("port", LinoValue::Int(18878)), + ]), + ), + ( + "models", + LinoValue::array([ + LinoValue::String("claude-haiku".to_string()), + LinoValue::String("claude-opus".to_string()), + ]), + ), + ]) +} + +#[test] +fn a_record_is_written_on_one_line() { + let line = encode_line(&log_record()); + assert!( + !line.contains('\n') && !line.contains('\r'), + "a record must stay on one line, got {line:?}" + ); + assert_eq!( + line, + r#"(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1") (port 18878))) (models ("claude-haiku" "claude-opus")))"# + ); +} + +#[test] +fn a_line_is_valid_links_notation() { + let line = encode_line(&log_record()); + parse_lino(&line).unwrap_or_else(|error| { + panic!("the notation's own parser rejected {line:?}: {error:?}"); + }); +} + +/// The dialect a downstream project invented for the same need, which the +/// notation's parser rejects -- the reason this format exists. +#[test] +fn the_hand_rolled_dialect_is_the_one_the_parser_rejects() { + assert!(parse_lino(r#"((:"bytes" 2827) (:"complete" true))"#).is_err()); +} + +#[test] +fn both_forms_of_the_same_value_decode_alike() { + for value in [ + log_record(), + LinoValue::Array(vec![]), + LinoValue::Object(vec![]), + LinoValue::array([LinoValue::Object(vec![]), LinoValue::Array(vec![])]), + LinoValue::object([("empty", LinoValue::Array(vec![]))]), + LinoValue::Int(42), + LinoValue::Null, + ] { + assert_eq!( + decode(&encode_line(&value)), + decode(&encode(&value)), + "the two forms disagree about {value:?}" + ); + assert_eq!(decode_line(&encode_line(&value)), Ok(value)); + } +} + +#[test] +fn a_string_keeps_its_own_characters_on_one_line() { + let value = LinoValue::object([( + "text", + LinoValue::String("quote \" backslash \\ ünïcödé".to_string()), + )]); + let line = encode_line(&value); + assert_eq!(line, r#"(o: (text 'quote " backslash \ ünïcödé'))"#); + assert_eq!(decode_line(&line), Ok(value)); +} + +/// A newline inside a string would end the record, so such a string is the one +/// thing written encoded -- individually, so the rest of the record stays +/// readable. +#[test] +fn a_string_holding_a_newline_still_fits_on_one_line() { + let value = LinoValue::object([ + ("readable", LinoValue::String("still visible".to_string())), + ("multiline", LinoValue::String("line1\nline2".to_string())), + ]); + let line = encode_line(&value); + assert_eq!( + line, + r#"(o: (readable "still visible") (multiline (base64 "bGluZTEKbGluZTI=")))"# + ); + assert!( + !line.contains('\n'), + "a record must stay on one line: {line:?}" + ); + assert_eq!(decode_line(&line), Ok(value)); +} + +/// The one ambiguity a flat layout has: is `(a 1)` a one-pair object or a +/// two-element array? On one line an object says so with the `o:` marker, so +/// both values keep their own spelling. +#[test] +fn a_one_pair_object_is_not_a_two_element_array() { + let object = LinoValue::object([("a", LinoValue::Int(1))]); + let array = LinoValue::array([LinoValue::String("a".to_string()), LinoValue::Int(1)]); + + assert_eq!(encode_line(&object), "(o: (a 1))"); + assert_eq!(encode_line(&array), r#"("a" 1)"#); + assert_eq!(decode_line("(o: (a 1))"), Ok(object)); + assert_eq!(decode_line(r#"("a" 1)"#), Ok(array)); +} + +/// Because the marker answers it, the empty key round-trips instead of being +/// rejected: `("" 2)` is a pair like any other inside a marked object. +#[test] +fn the_empty_key_survives_the_round_trip() { + let value = LinoValue::object([("", LinoValue::Int(2))]); + assert_eq!(encode_line(&value), r#"(o: ("" 2))"#); + assert_eq!(decode_line(&encode_line(&value)), Ok(value)); +} + +#[test] +fn a_marked_object_holding_something_that_is_not_a_pair_is_rejected() { + let error = decode_line("(o: 1 2)").expect_err("a marked object holds pairs only"); + assert!( + error.to_string().contains("pairs"), + "the error must say what a marked object holds, got {error}" + ); +} + +/// Reading a log means handing over one record at a time, so a decoder that +/// silently accepted two lines would merge two records into one value. +#[test] +fn several_lines_are_not_one_record() { + assert!(decode_line("(o: (a 1))\n(o: (b 2))").is_err()); +} + +/// A trailing newline is what a line reader may keep, so it is trimmed rather +/// than refused. +#[test] +fn a_trailing_newline_is_not_a_second_record() { + assert_eq!( + decode_line("(o: (a 1))\n"), + Ok(LinoValue::object([("a", LinoValue::Int(1))])) + ); +} + +/// A line whose first value is null is a readable line, not the compact null: +/// `decode` must not route it to the base64 reader. +#[test] +fn a_line_starting_with_null_is_still_read_as_a_line() { + assert_eq!( + decode("(null 1)"), + Ok(LinoValue::array([LinoValue::Null, LinoValue::Int(1)])) + ); + assert_eq!( + decode("(o: (a null))"), + Ok(LinoValue::object([("a", LinoValue::Null)])) + ); + // The one document both forms claim: `(null)` is the compact null, and stays + // read that way, so documents written before this format keep decoding. + assert_eq!(decode("(null)"), Ok(LinoValue::Null)); +} From 9351af14d25571e68df2dd5e83e00d40e702d7e5 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 27 Aug 2026 12:00:50 +0000 Subject: [PATCH 05/12] feat(js): add readable single-line encodeLine/decodeLine --- js/src/codec.js | 110 +++++++++++-- js/src/index.js | 4 + js/src/readable.js | 183 ++++++++++++++++++++- js/tests/test_readable_conformance.test.js | 29 +++- js/tests/test_single_line_format.test.js | 116 +++++++++++++ 5 files changed, 421 insertions(+), 21 deletions(-) create mode 100644 js/tests/test_single_line_format.test.js diff --git a/js/src/codec.js b/js/src/codec.js index d457d42..520d0a6 100644 --- a/js/src/codec.js +++ b/js/src/codec.js @@ -99,6 +99,39 @@ export class ObjectCodec { return readable.encode(obj, indent); } + /** + * Encode a JavaScript object to the readable, single-line Links Notation + * format. + * + * The result never contains a newline, so one value is one line: an + * append-only log written this way stays greppable, tailable and countable by + * `wc -l`. See `readable.js` for the exact shape. + * + * @param {Object} options - Options + * @param {*} options.obj - The JavaScript object to encode + * @returns {string} One line of readable Links Notation + */ + encodeLine(options = {}) { + const { obj } = options; + return readable.encodeLine(obj); + } + + /** + * Decode one line of the readable, single-line Links Notation format. + * + * This is the exact inverse of {@link ObjectCodec#encodeLine}. Input spanning + * more than one line is rejected, so two log records never merge into one + * value. + * + * @param {Object} options - Options + * @param {string} options.notation - One line of readable Links Notation + * @returns {*} Reconstructed JavaScript object + */ + decodeLine(options = {}) { + const { notation } = options; + return readable.decodeLine(notation); + } + /** * Encode a JavaScript object to the compact, single-line Links Notation format. * @@ -605,6 +638,16 @@ const COMPACT_TYPE_MARKERS = new Set([ 'dict', ]); +/** + * Markers a compact document writes without a payload, so `(null)` is a compact + * null while `(null 1)` is a readable line holding two values. + */ +const EMPTY_BODY_MARKERS = new Set([ + ObjectCodec.TYPE_NULL, + ObjectCodec.TYPE_UNDEFINED, + 'None', +]); + /** * Whether a document is in the compact (base64) format rather than the readable * one. The compact format always opens with `(` followed by a type marker, @@ -623,28 +666,40 @@ export function isCompactNotation(notation) { return false; } - const tokens = firstLine - .slice(1) - .split(/[\s()]+/) - .filter((token) => token.length > 0); - - let marker = tokens[0]; - if (marker === undefined) { - return false; - } + // A compact document names the type of its value first, so a link that opens + // another link straight away is the readable form, whose links nest. + let [marker, rest] = splitToken(firstLine.slice(1).trimStart()); // Skip the `obj_N:` definition id, if present. if (marker.endsWith(':')) { if (!marker.startsWith('obj_')) { return false; } - marker = tokens[1]; - if (marker === undefined) { - return false; - } + [marker, rest] = splitToken(rest.trimStart()); + } + + if (!COMPACT_TYPE_MARKERS.has(marker)) { + return false; + } + + // A compact null is the whole link: `(null)`. A link that holds more than the + // marker is a readable line whose first value happens to be null. + if (EMPTY_BODY_MARKERS.has(marker)) { + return rest.trimStart().startsWith(')'); } - return COMPACT_TYPE_MARKERS.has(marker); + return true; +} + +/** + * Split off the first token of a link body: the text up to the next whitespace + * or parenthesis. A body that opens with a parenthesis has no token of its own. + * @param {string} input - The link body + * @returns {[string, string]} The token and the text after it + */ +function splitToken(input) { + const end = input.search(/[\s()]/); + return end === -1 ? [input, ''] : [input.slice(0, end), input.slice(end)]; } // Convenience functions @@ -661,6 +716,33 @@ export function encode(options = {}) { return _defaultCodec.encode(options); } +/** + * Encode a JavaScript object to the readable, single-line Links Notation format. + * + * The result never contains a newline, so one value is one line of an + * append-only log. + * + * @param {Object} options - Options + * @param {*} options.obj - The JavaScript object to encode + * @returns {string} One line of readable Links Notation + */ +export function encodeLine(options = {}) { + return _defaultCodec.encodeLine(options); +} + +/** + * Decode one line of the readable, single-line Links Notation format. + * + * The exact inverse of {@link encodeLine}. + * + * @param {Object} options - Options + * @param {string} options.notation - One line of readable Links Notation + * @returns {*} Reconstructed JavaScript object + */ +export function decodeLine(options = {}) { + return _defaultCodec.decodeLine(options); +} + /** * Encode a JavaScript object to the compact, single-line Links Notation format. * diff --git a/js/src/index.js b/js/src/index.js index ccbf01c..d7a99bd 100644 --- a/js/src/index.js +++ b/js/src/index.js @@ -3,6 +3,7 @@ * * This library provides: * - Readable recursive indented Links Notation for JSON-style repository data + * - Readable single-line Links Notation for append-only logs, one record per line * - Typed serialization/deserialization for exact JavaScript object graphs * - Typed support for circular references and shared object identity * - JSON to Links Notation conversion utilities @@ -20,6 +21,8 @@ export { ObjectCodec, encode, + encodeLine, + decodeLine, encodeCompact, encodeObfuscated, decode, @@ -31,6 +34,7 @@ export { export { DEFAULT_INDENT, BASE64_MARKER, + OBJECT_MARKER, CircularReferenceError, } from './readable.js'; diff --git a/js/src/readable.js b/js/src/readable.js index 9be5b6b..2d2150b 100644 --- a/js/src/readable.js +++ b/js/src/readable.js @@ -42,6 +42,34 @@ * CRLF normalisation would corrupt) are marked individually as * `(base64 "…")` instead of encoding the whole document. * + * # Single-line form + * + * {@link encodeLine} writes the same document on one line, so one record is one + * line and an append-only log stays greppable, tailable and countable by + * `wc -l`. Rows can no longer be told apart by line breaks there, so an object + * names itself with the `o` link id the notation already has, and its pairs are + * written as their own links: + * + * ```text + * (o: (type "RouterState") (server (o: (host "127.0.0.1") (port 18878))) (models ("claude-haiku" "claude-opus"))) + * ``` + * + * | Value | Single-line form | + * | ---------------- | ------------------------------- | + * | plain object | `(o: (key value) …)` | + * | empty object | `(o:)` | + * | `Array` | `(value …)` | + * | empty `Array` | `()` | + * | scalars | exactly as in the indented form | + * + * The marker is what answers the ambiguity a flat layout otherwise has: without + * it `((key value))` reads both as the one-pair object and as the array holding + * the two-element array, and an empty key makes it worse. With it, a bare `( )` + * is always an array and a marked one is always an object, so every value — + * empty key included — survives the round trip. Consequently a *hand-written* + * one-line link such as `(a 1)` is the two-element array, not the one-pair + * object: on one line, objects say so. + * * @module readable */ @@ -53,6 +81,9 @@ export const DEFAULT_INDENT = ' '; /** Marker used for values that cannot be represented as plain text. */ export const BASE64_MARKER = 'base64'; +/** Link id naming an object in the single-line form, written as `(o: …)`. */ +export const OBJECT_MARKER = 'o'; + /** Literals that a bare reference decodes to instead of a string. */ const BARE_LITERALS = new Map([ ['null', null], @@ -99,6 +130,40 @@ export function encode(value, indent = DEFAULT_INDENT) { return out.join(''); } +/** + * Encode a value into the readable, single-line Links Notation form. + * + * The result never contains a newline, so one value is one line of an + * append-only log. See the module documentation for the shape. + * @param {*} value - The value to encode + * @returns {string} The readable Links Notation document, on one line + * @throws {CircularReferenceError} If the value refers back to itself + * @throws {TypeError} If the value holds a type this format cannot write + */ +export function encodeLine(value) { + const out = []; + writeLineValue(value, out, new Set()); + return out.join(''); +} + +/** + * Decode the readable, single-line Links Notation form back into a value. + * + * This is the exact inverse of {@link encodeLine}. Input spanning more than one + * line is rejected: a line-based reader hands over one record at a time, and + * silently accepting several would merge two records into one value. + * @param {string} text - One line of a readable Links Notation document + * @returns {*} The reconstructed value + * @throws {SyntaxError} If the input holds more than one line + */ +export function decodeLine(text) { + const line = text.replace(/^[\n\r]+/, '').replace(/[\n\r]+$/, ''); + if (line.includes('\n') || line.includes('\r')) { + throw new SyntaxError('a single-line document cannot contain a line break'); + } + return decode(line); +} + /** * Decode the readable, indented Links Notation form back into a value. * @param {string} text - The readable Links Notation document @@ -119,7 +184,7 @@ export function decode(text) { return nodeToValue(rows[0][0]); } - return rowsToValue(rows, true); + return rowsToValue(rows, true, false); } // === Encoding === @@ -157,6 +222,51 @@ function writeValue(value, indent, level, out, path) { out.push(formatScalar(value)); } +/** + * Write a value on one line. Objects name themselves with the `o` link id and + * write each pair as its own link, so nothing depends on where lines break. + * @param {*} value - The value to write + * @param {string[]} out - Output chunks, appended in place + * @param {Set} path - Containers currently being written + */ +function writeLineValue(value, out, path) { + if (Array.isArray(value)) { + enterPath(value, path); + out.push('('); + value.forEach((item, index) => { + if (index > 0) { + out.push(' '); + } + writeLineValue(item, out, path); + }); + out.push(')'); + path.delete(value); + return; + } + + if (isPlainContainer(value)) { + enterPath(value, path); + const entries = Object.entries(value); + if (entries.length === 0) { + // `()` is the empty array, so the empty object keeps its marker. + out.push(`(${OBJECT_MARKER}:)`); + path.delete(value); + return; + } + out.push(`(${OBJECT_MARKER}:`); + for (const [key, child] of entries) { + out.push(` (${formatKey(key)} `); + writeLineValue(child, out, path); + out.push(')'); + } + out.push(')'); + path.delete(value); + return; + } + + out.push(formatScalar(value)); +} + /** * Mark a container as being written, so a reference back to it is caught. * @@ -471,14 +581,33 @@ class Cursor { if (token.kind === TOKEN_OPEN) { this.pos += 1; + const object = this.takeObjectMarker(); const multiline = this.linkIsMultiline(); const rows = this.parseRows(false); - return { ref: false, rows, multiline }; + return { ref: false, rows, multiline, object }; } throw new SyntaxError('unexpected token in readable notation'); } + /** + * Consume the `o:` marker if the link that just opened carries one, which is + * how the single-line form says "this link is an object, not an array". + * @returns {boolean} True when the marker was there and was consumed + */ + takeObjectMarker() { + const token = this.tokens[this.pos]; + const isMarker = + token !== undefined && + token.kind === TOKEN_REF && + !token.quoted && + token.value === `${OBJECT_MARKER}:`; + if (isMarker) { + this.pos += 1; + } + return isMarker; + } + /** * Whether the link that just opened spans more than one line, which is what * tells an empty object (`(\n)`) from an empty array (`()`). @@ -500,10 +629,14 @@ class Cursor { function nodeToValue(node) { return node.ref ? refToValue(node.value, node.quoted) - : rowsToValue(node.rows, node.multiline); + : rowsToValue(node.rows, node.multiline, node.object); } -function rowsToValue(rows, multiline) { +function rowsToValue(rows, multiline, objectMarker) { + if (objectMarker) { + return markedObjectToValue(rows); + } + if (rows.length === 0) { return multiline ? {} : []; } @@ -513,6 +646,12 @@ function rowsToValue(rows, multiline) { return marked.value; } + // Written on one line, a link is a list of values: an object on one line says + // so with the `o:` marker, which is what keeps `(key value)` unambiguous. + if (!multiline) { + return rows.flatMap((row) => row.map(nodeToValue)); + } + // `key value` on every line makes an object; anything else is a list of values. const isObject = rows.every((row) => row.length === 2 && row[0].ref); @@ -533,6 +672,42 @@ function rowsToValue(rows, multiline) { return items; } +/** + * Build the object a `(o: (key value) …)` link describes. Every value in it is + * a pair, so anything else is a malformed document rather than a silent array. + * @param {Array>} rows - The rows of the marked link + * @returns {object} The reconstructed object + * @throws {SyntaxError} If the link holds anything that is not a pair + */ +function markedObjectToValue(rows) { + const result = {}; + + for (const node of rows.flat()) { + if (node.ref || node.object) { + throw new SyntaxError( + `an object marked '${OBJECT_MARKER}:' holds (key value) pairs, ` + + 'found a value that is not a pair' + ); + } + if (node.rows.length !== 1) { + throw new SyntaxError( + `an object marked '${OBJECT_MARKER}:' holds (key value) pairs, ` + + `found a link of ${node.rows.length} lines` + ); + } + const [row] = node.rows; + if (row.length !== 2 || !row[0].ref) { + throw new SyntaxError( + `an object marked '${OBJECT_MARKER}:' holds (key value) pairs, ` + + `found a link of ${row.length} values` + ); + } + result[row[0].value] = nodeToValue(row[1]); + } + + return result; +} + /** * Recognise `(base64 "…")`, the individual marker for values that could not be * written as text. A quoted `base64` key is an ordinary object key, not a marker. diff --git a/js/tests/test_readable_conformance.test.js b/js/tests/test_readable_conformance.test.js index 5b96740..8a976a1 100644 --- a/js/tests/test_readable_conformance.test.js +++ b/js/tests/test_readable_conformance.test.js @@ -3,8 +3,9 @@ * * The cases live in `fixtures/readable-format/cases.json` at the repository root * and are shared by the JavaScript, Python, Rust and C# suites: every - * implementation has to encode the same value to exactly the same text, which is - * what keeps the four outputs byte-identical. + * implementation has to encode the same value to exactly the same text and to + * exactly the same single line, which is what keeps the four outputs + * byte-identical. */ import { test } from 'node:test'; @@ -12,7 +13,7 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { dirname, join } from 'node:path'; -import { encode, decode } from '../src/index.js'; +import { encode, encodeLine, decode, decodeLine } from '../src/index.js'; const LANGUAGE = 'js'; const LANGUAGES = new Set(['js', 'python', 'rust', 'csharp']); @@ -120,4 +121,26 @@ for (const testCase of cases) { `${JSON.stringify(decode({ notation: testCase.text }))} != ${JSON.stringify(build(testCase.value))}` ); }); + + test(`encodeLine matches the shared line: ${testCase.name}`, () => { + assert.equal(encodeLine({ obj: build(testCase.value) }), testCase.line); + }); + + test(`the shared line holds no line break: ${testCase.name}`, () => { + assert.ok(!/[\n\r]/.test(testCase.line), testCase.line); + }); + + test(`decodeLine matches the shared value: ${testCase.name}`, () => { + assert.ok( + same(decodeLine({ notation: testCase.line }), build(testCase.value)), + `${JSON.stringify(decodeLine({ notation: testCase.line }))} != ${JSON.stringify(build(testCase.value))}` + ); + }); + + test(`decode reads the shared line too: ${testCase.name}`, () => { + assert.ok( + same(decode({ notation: testCase.line }), build(testCase.value)), + `${JSON.stringify(decode({ notation: testCase.line }))} != ${JSON.stringify(build(testCase.value))}` + ); + }); } diff --git a/js/tests/test_single_line_format.test.js b/js/tests/test_single_line_format.test.js new file mode 100644 index 0000000..c1739be --- /dev/null +++ b/js/tests/test_single_line_format.test.js @@ -0,0 +1,116 @@ +/** + * Tests for the readable, single-line format produced by `encodeLine()` + * (issue #43). + * + * An append-only log wants one record per line: appending is one write, + * compaction cuts at a newline, and `grep`, `tail -f` and `wc -l` all treat a + * line as an event. `encode()` spreads a record over many lines and + * `encodeCompact()` hides it in base64, so neither serves that reader. + */ + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { Parser } from 'links-notation'; +import { encode, encodeLine, decode, decodeLine } from '../src/index.js'; + +/** A record of the shape an append-only log actually holds. */ +const LOG_RECORD = { + bytes: 2827, + complete: true, + server: { host: '127.0.0.1', port: 18878 }, + models: ['claude-haiku', 'claude-opus'], +}; + +const LOG_RECORD_LINE = + '(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1")' + + ' (port 18878))) (models ("claude-haiku" "claude-opus")))'; + +test('a record is written on one line', () => { + const line = encodeLine({ obj: LOG_RECORD }); + assert.ok(!/[\n\r]/.test(line), line); + assert.equal(line, LOG_RECORD_LINE); +}); + +test('a line is valid links notation', () => { + new Parser().parse(encodeLine({ obj: LOG_RECORD })); +}); + +test('the hand-rolled dialect is the one the parser rejects', () => { + assert.throws(() => + new Parser().parse('((:"bytes" 2827) (:"complete" true))') + ); +}); + +test('both forms of the same value decode alike', () => { + const values = [ + LOG_RECORD, + [], + {}, + [{}, []], + { empty: [] }, + 42, + null, + 'text', + ]; + for (const value of values) { + assert.deepEqual( + decode({ notation: encodeLine({ obj: value }) }), + decode({ notation: encode({ obj: value }) }) + ); + assert.deepEqual( + decodeLine({ notation: encodeLine({ obj: value }) }), + value + ); + } +}); + +test('a string keeps its own characters on one line', () => { + const value = { text: 'quote " backslash \\ ünïcödé' }; + const line = encodeLine({ obj: value }); + assert.equal(line, `(o: (text 'quote " backslash \\ ünïcödé'))`); + assert.deepEqual(decodeLine({ notation: line }), value); +}); + +test('a string holding a newline still fits on one line', () => { + const value = { readable: 'still visible', multiline: 'line1\nline2' }; + const line = encodeLine({ obj: value }); + assert.equal( + line, + '(o: (readable "still visible") (multiline (base64 "bGluZTEKbGluZTI=")))' + ); + assert.ok(!/[\n\r]/.test(line), line); + assert.deepEqual(decodeLine({ notation: line }), value); +}); + +test('a one-pair object is not a two-element array', () => { + assert.equal(encodeLine({ obj: { a: 1 } }), '(o: (a 1))'); + assert.equal(encodeLine({ obj: ['a', 1] }), '("a" 1)'); + assert.deepEqual(decodeLine({ notation: '(o: (a 1))' }), { a: 1 }); + assert.deepEqual(decodeLine({ notation: '("a" 1)' }), ['a', 1]); +}); + +test('the empty key survives the round trip', () => { + const value = { '': 2 }; + assert.equal(encodeLine({ obj: value }), `(o: ("" 2))`); + assert.deepEqual(decodeLine({ notation: encodeLine({ obj: value }) }), value); +}); + +test('a marked object holding something that is not a pair is rejected', () => { + assert.throws(() => decodeLine({ notation: '(o: 1 2)' }), /pairs/); +}); + +test('several lines are not one record', () => { + assert.throws(() => decodeLine({ notation: '(o: (a 1))\n(o: (b 2))' })); +}); + +test('a trailing newline is not a second record', () => { + assert.deepEqual(decodeLine({ notation: '(o: (a 1))\n' }), { a: 1 }); +}); + +test('a line starting with null is still read as a line', () => { + assert.deepEqual(decode({ notation: '(null 1)' }), [null, 1]); + assert.deepEqual(decode({ notation: '(o: (a null))' }), { a: null }); + // The one document both forms claim: `(null)` is the compact null, and stays + // read that way, so documents written before this format keep decoding. + assert.equal(decode({ notation: '(null)' }), null); +}); From aed77117a82cf49c6743eddb88c6787a867ad553 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 27 Aug 2026 12:07:24 +0000 Subject: [PATCH 06/12] feat(python): write a record on one line with encode_line/decode_line --- .../link_notation_objects_codec/__init__.py | 8 + .../src/link_notation_objects_codec/codec.py | 107 ++++++++++- .../link_notation_objects_codec/readable.py | 174 +++++++++++++++++- python/tests/test_readable_conformance.py | 31 +++- python/tests/test_single_line_format.py | 124 +++++++++++++ 5 files changed, 427 insertions(+), 17 deletions(-) create mode 100644 python/tests/test_single_line_format.py diff --git a/python/src/link_notation_objects_codec/__init__.py b/python/src/link_notation_objects_codec/__init__.py index dfc00f7..9a40c58 100644 --- a/python/src/link_notation_objects_codec/__init__.py +++ b/python/src/link_notation_objects_codec/__init__.py @@ -6,6 +6,8 @@ :func:`encode` writes the readable, indented format by default; :func:`decode` reads both that and the compact (base64) format written by earlier versions. +:func:`encode_line` writes the same readable document on one line, so an +append-only log holds one record per line. """ from importlib.metadata import PackageNotFoundError @@ -15,8 +17,10 @@ ObjectCodec, decode, decode_compact, + decode_line, encode, encode_compact, + encode_line, encode_obfuscated, is_compact_notation, ) @@ -30,6 +34,7 @@ from .readable import ( BASE64_MARKER, DEFAULT_INDENT, + OBJECT_MARKER, CircularReferenceError, ReadableFormatError, ) @@ -43,9 +48,11 @@ __all__ = [ "ObjectCodec", "encode", + "encode_line", "encode_compact", "encode_obfuscated", "decode", + "decode_line", "decode_compact", "is_compact_notation", "escape_reference", @@ -54,6 +61,7 @@ "parse_indented", "DEFAULT_INDENT", "BASE64_MARKER", + "OBJECT_MARKER", "ReadableFormatError", "CircularReferenceError", "DEBUG_ENV_VAR", diff --git a/python/src/link_notation_objects_codec/codec.py b/python/src/link_notation_objects_codec/codec.py index 9e76fb1..12e5878 100644 --- a/python/src/link_notation_objects_codec/codec.py +++ b/python/src/link_notation_objects_codec/codec.py @@ -44,6 +44,11 @@ ) +#: Markers a compact document writes without a payload, so ``(null)`` is a +#: compact null while ``(null 1)`` is a readable line holding two values. +_EMPTY_BODY_MARKERS: frozenset[str] = frozenset({"null", "None", "undefined"}) + + def is_compact_notation(notation: str) -> bool: """Whether a document is in the compact (type-tagged, base64) format. @@ -62,21 +67,35 @@ def is_compact_notation(notation: str) -> bool: if first_line is None or not first_line.startswith("("): return False - tokens = [token for token in re.split(r"[\s()]+", first_line[1:]) if token] - if not tokens: - return False - - marker = tokens[0] + # A compact document names the type of its value first, so a link that opens + # another link straight away is the readable form, whose links nest. + marker, rest = _split_token(first_line[1:].lstrip()) # Skip the ``obj_N:`` definition id, if present. if marker.endswith(":"): if not marker[:-1].startswith("obj_"): return False - if len(tokens) < 2: - return False - marker = tokens[1] + marker, rest = _split_token(rest.lstrip()) + + if marker not in _COMPACT_TYPE_MARKERS: + return False - return marker in _COMPACT_TYPE_MARKERS + # A compact null is the whole link: ``(null)``. A link that holds more than + # the marker is a readable line whose first value happens to be null. + if marker in _EMPTY_BODY_MARKERS: + return rest.lstrip().startswith(")") + + return True + + +def _split_token(text: str) -> tuple[str, str]: + """Split off the first token of a link body. + + The token is the text up to the next whitespace or parenthesis; a body that + opens with a parenthesis has no token of its own. + """ + match = re.search(r"[\s()]", text) + return (text, "") if match is None else (text[: match.start()], text[match.start() :]) class ObjectCodec: @@ -183,6 +202,37 @@ def encode(self, obj: Any, indent: str = readable.DEFAULT_INDENT) -> str: """ return readable.encode(obj, indent) + def encode_line(self, obj: Any) -> str: + """ + Encode a Python object to the readable, single-line Links Notation format. + + The result never contains a newline, so one value is one line: an + append-only log written this way stays greppable, tailable and countable + by ``wc -l``. See :mod:`link_notation_objects_codec.readable` for the shape. + + Args: + obj: The Python object to encode + + Returns: + One line of readable Links Notation + """ + return readable.encode_line(obj) + + def decode_line(self, notation: str) -> Any: + """ + Decode one line of the readable, single-line Links Notation format. + + This is the exact inverse of :meth:`encode_line`. Input spanning more + than one line is rejected, so two log records never merge into one value. + + Args: + notation: One line of readable Links Notation + + Returns: + Reconstructed Python object + """ + return readable.decode_line(notation) + def encode_compact(self, obj: Any) -> str: """ Encode a Python object to the compact, single-line Links Notation format. @@ -589,6 +639,45 @@ def encode(obj: Any, indent: str = readable.DEFAULT_INDENT) -> str: return _default_codec.encode(obj, indent) +def encode_line(obj: Any) -> str: + """ + Encode a Python object to the readable, single-line Links Notation format. + + The result never contains a newline, so one value is one line of an + append-only log. + + Args: + obj: The Python object to encode + + Returns: + One line of readable Links Notation + + Example: + >>> encode_line({"age": 30}) + '(o: (age 30))' + """ + return _default_codec.encode_line(obj) + + +def decode_line(notation: str) -> Any: + """ + Decode one line of the readable, single-line Links Notation format. + + The exact inverse of :func:`encode_line`. + + Args: + notation: One line of readable Links Notation + + Returns: + Reconstructed Python object + + Example: + >>> decode_line('(o: (age 30))') + {'age': 30} + """ + return _default_codec.decode_line(notation) + + def encode_compact(obj: Any) -> str: """ Encode a Python object to the compact, single-line Links Notation format. diff --git a/python/src/link_notation_objects_codec/readable.py b/python/src/link_notation_objects_codec/readable.py index 19bfa35..16f1898 100644 --- a/python/src/link_notation_objects_codec/readable.py +++ b/python/src/link_notation_objects_codec/readable.py @@ -42,6 +42,35 @@ control characters (including newlines and tabs, which line-based tooling and CRLF normalisation would corrupt) are marked individually as ``(base64 "...")`` instead of encoding the whole document. + +Single-line form +---------------- + +:func:`encode_line` writes the same document on one line, so one record is one +line and an append-only log stays greppable, tailable and countable by ``wc -l``. +Rows can no longer be told apart by line breaks there, so a dict names itself +with the ``o`` link id the notation already has, and its pairs are written as +their own links:: + + (o: (type "RouterState") (server (o: (host "127.0.0.1") (port 18878)))) + +================== =============================== +Value Single-line form +================== =============================== +``dict`` ``(o: (key value) ...)`` +empty ``dict`` ``(o:)`` +``list`` ``(value ...)`` +empty ``list`` ``()`` +scalars exactly as in the indented form +================== =============================== + +The marker is what answers the ambiguity a flat layout otherwise has: without it +``((key value))`` reads both as the one-pair dict and as the list holding the +two-element list, and an empty key makes it worse. With it, a bare ``( )`` is +always a list and a marked one is always a dict, so every value -- empty key +included -- survives the round trip. Consequently a *hand-written* one-line link +such as ``(a 1)`` is the two-element list, not the one-pair dict: on one line, +dicts say so. """ import base64 @@ -60,6 +89,9 @@ #: Marker used for values that cannot be represented as plain text. BASE64_MARKER = "base64" +#: Link id naming a dict in the single-line form, written as ``(o: ...)``. +OBJECT_MARKER = "o" + #: Quote characters that open a quoted reference. _QUOTE_CHARS = ("'", '"', "`") @@ -102,6 +134,49 @@ def encode(value: Any, indent: str = DEFAULT_INDENT) -> str: return "".join(out) +def encode_line(value: Any) -> str: + """Encode a value into the readable, single-line Links Notation form. + + The result never contains a newline, so one value is one line of an + append-only log. See the module documentation for the shape. + + Args: + value: The value to encode. + + Returns: + The readable Links Notation document, on one line. + + Raises: + CircularReferenceError: If the value refers back to itself. + TypeError: If the value holds a type this format cannot write. + """ + out: list[str] = [] + _write_line_value(value, out, set()) + return "".join(out) + + +def decode_line(text: str) -> Any: + """Decode the readable, single-line Links Notation form back into a value. + + This is the exact inverse of :func:`encode_line`. Input spanning more than + one line is rejected: a line-based reader hands over one record at a time, + and silently accepting several would merge two records into one value. + + Args: + text: One line of a readable Links Notation document. + + Returns: + The reconstructed value. + + Raises: + ReadableFormatError: If the input holds more than one line. + """ + line = text.strip("\n\r") + if "\n" in line or "\r" in line: + raise ReadableFormatError("a single-line document cannot contain a line break") + return decode(line) + + def decode(text: str) -> Any: """Decode the readable, indented Links Notation form back into a value. @@ -126,7 +201,7 @@ def decode(text: str) -> Any: if len(rows) == 1 and len(rows[0]) == 1: return _node_to_value(rows[0][0]) - return _rows_to_value(rows, multiline=True) + return _rows_to_value(rows, multiline=True, object_marker=False) # === Encoding === @@ -165,6 +240,41 @@ def write_item(item: Any) -> None: out.append(_format_scalar(value)) +def _write_line_value(value: Any, out: list[str], path: set[int]) -> None: + """Write a value on one line. + + Dicts name themselves with the ``o`` link id and write each pair as its own + link, so nothing depends on where lines break. + """ + if isinstance(value, dict): + with _on_path(value, path): + items = list(value.items()) + if not items: + # ``()`` is the empty list, so the empty dict keeps its marker. + out.append(f"({OBJECT_MARKER}:)") + return + + out.append(f"({OBJECT_MARKER}:") + for key, child in items: + out.append(f" ({_format_key(key)} ") + _write_line_value(child, out, path) + out.append(")") + out.append(")") + return + + if isinstance(value, (list, tuple, set, frozenset)): + with _on_path(value, path): + out.append("(") + for index, item in enumerate(value): + if index: + out.append(" ") + _write_line_value(item, out, path) + out.append(")") + return + + out.append(_format_scalar(value)) + + @contextmanager def _on_path(value: Any, path: set[int]) -> Iterator[None]: """Mark a container as being written, so a reference back to it is caught. @@ -313,7 +423,7 @@ class _Node: """A parsed element: a reference (remembering whether it was quoted, which is what distinguishes a string from a number) or a link.""" - __slots__ = ("is_ref", "value", "quoted", "rows", "multiline") + __slots__ = ("is_ref", "value", "quoted", "rows", "multiline", "is_object") def __init__( self, @@ -322,12 +432,14 @@ def __init__( quoted: bool = False, rows: list[list["_Node"]] | None = None, multiline: bool = False, + is_object: bool = False, ) -> None: self.is_ref = is_ref self.value = value self.quoted = quoted self.rows = rows if rows is not None else [] self.multiline = multiline + self.is_object = is_object def _tokenize(text: str) -> list[_Token]: @@ -438,12 +550,26 @@ def parse_node(self) -> _Node: if token.kind == _TOKEN_OPEN: self.pos += 1 + is_object = self._take_object_marker() multiline = self._link_is_multiline() rows = self.parse_rows(top_level=False) - return _Node(False, rows=rows, multiline=multiline) + return _Node(False, rows=rows, multiline=multiline, is_object=is_object) raise ReadableFormatError("unexpected token in readable notation") + def _take_object_marker(self) -> bool: + """Consume the ``o:`` marker if the link that just opened carries one, + which is how the single-line form says "this link is a dict, not a list".""" + if self.pos >= len(self.tokens): + return False + token = self.tokens[self.pos] + is_marker = ( + token.kind == _TOKEN_REF and not token.quoted and token.value == f"{OBJECT_MARKER}:" + ) + if is_marker: + self.pos += 1 + return is_marker + def _link_is_multiline(self) -> bool: """Whether the link that just opened spans more than one line, which is what tells an empty dict (``(\\n)``) from an empty list (``()``).""" @@ -458,10 +584,13 @@ def _link_is_multiline(self) -> bool: def _node_to_value(node: _Node) -> Any: if node.is_ref: return _ref_to_value(node.value, node.quoted) - return _rows_to_value(node.rows, node.multiline) + return _rows_to_value(node.rows, node.multiline, node.is_object) + +def _rows_to_value(rows: list[list[_Node]], multiline: bool, object_marker: bool) -> Any: + if object_marker: + return _marked_object_to_value(rows) -def _rows_to_value(rows: list[list[_Node]], multiline: bool) -> Any: if not rows: return {} if multiline else [] @@ -469,6 +598,11 @@ def _rows_to_value(rows: list[list[_Node]], multiline: bool) -> Any: if marked is not None: return marked[0] + # Written on one line, a link is a list of values: a dict on one line says so + # with the ``o:`` marker, which is what keeps ``(key value)`` unambiguous. + if not multiline: + return [_node_to_value(node) for row in rows for node in row] + # ``key value`` on every line makes a dict; anything else is a list of values. is_dict = all(len(row) == 2 and row[0].is_ref for row in rows) @@ -485,6 +619,36 @@ def _rows_to_value(rows: list[list[_Node]], multiline: bool) -> Any: return items +def _marked_object_to_value(rows: list[list[_Node]]) -> dict[str, Any]: + """Build the dict a ``(o: (key value) ...)`` link describes. + + Every value in it is a pair, so anything else is a malformed document rather + than a silent list. + """ + result: dict[str, Any] = {} + + for node in (node for row in rows for node in row): + if node.is_ref or node.is_object: + raise ReadableFormatError( + f"an object marked '{OBJECT_MARKER}:' holds (key value) pairs, " + "found a value that is not a pair" + ) + if len(node.rows) != 1: + raise ReadableFormatError( + f"an object marked '{OBJECT_MARKER}:' holds (key value) pairs, " + f"found a link of {len(node.rows)} lines" + ) + row = node.rows[0] + if len(row) != 2 or not row[0].is_ref: + raise ReadableFormatError( + f"an object marked '{OBJECT_MARKER}:' holds (key value) pairs, " + f"found a link of {len(row)} values" + ) + result[row[0].value] = _node_to_value(row[1]) + + return result + + def _decode_marked_value(rows: list[list[_Node]]) -> tuple[str] | None: """Recognise ``(base64 "...")``, the individual marker for values that could not be written as text. diff --git a/python/tests/test_readable_conformance.py b/python/tests/test_readable_conformance.py index 706ee16..84af1ef 100644 --- a/python/tests/test_readable_conformance.py +++ b/python/tests/test_readable_conformance.py @@ -2,8 +2,9 @@ The cases live in ``fixtures/readable-format/cases.json`` at the repository root and are shared by the JavaScript, Python, Rust and C# suites: every -implementation has to encode the same value to exactly the same text, which is -what keeps the four outputs byte-identical. +implementation has to encode the same value to exactly the same text and to +exactly the same single line, which is what keeps the four outputs +byte-identical. """ import json @@ -13,7 +14,7 @@ import pytest -from link_notation_objects_codec import decode, encode +from link_notation_objects_codec import decode, decode_line, encode, encode_line LANGUAGE = "python" @@ -83,3 +84,27 @@ def test_decode_matches_the_shared_value(case: dict[str, Any]) -> None: expected = _build(case["value"]) decoded = decode(case["text"]) assert _same(decoded, expected), f"{decoded!r} != {expected!r}" + + +@pytest.mark.parametrize("case", ACTIVE, ids=lambda case: case["name"]) +def test_encode_line_matches_the_shared_line(case: dict[str, Any]) -> None: + assert encode_line(_build(case["value"])) == case["line"] + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: case["name"]) +def test_the_shared_line_holds_no_line_break(case: dict[str, Any]) -> None: + assert "\n" not in case["line"] and "\r" not in case["line"], case["line"] + + +@pytest.mark.parametrize("case", ACTIVE, ids=lambda case: case["name"]) +def test_decode_line_matches_the_shared_value(case: dict[str, Any]) -> None: + expected = _build(case["value"]) + decoded = decode_line(case["line"]) + assert _same(decoded, expected), f"{decoded!r} != {expected!r}" + + +@pytest.mark.parametrize("case", ACTIVE, ids=lambda case: case["name"]) +def test_decode_reads_the_shared_line_too(case: dict[str, Any]) -> None: + expected = _build(case["value"]) + decoded = decode(case["line"]) + assert _same(decoded, expected), f"{decoded!r} != {expected!r}" diff --git a/python/tests/test_single_line_format.py b/python/tests/test_single_line_format.py new file mode 100644 index 0000000..a6887ff --- /dev/null +++ b/python/tests/test_single_line_format.py @@ -0,0 +1,124 @@ +"""Tests for the readable, single-line format produced by ``encode_line()`` +(issue #43). + +An append-only log wants one record per line: appending is one write, compaction +cuts at a newline, and ``grep``, ``tail -f`` and ``wc -l`` all treat a line as an +event. ``encode()`` spreads a record over many lines and ``encode_compact()`` +hides it in base64, so neither serves that reader. +""" + +from typing import Any + +import pytest +from links_notation import Parser + +from link_notation_objects_codec import ( + ReadableFormatError, + decode, + decode_line, + encode, + encode_line, +) + +#: A record of the shape an append-only log actually holds. +LOG_RECORD = { + "bytes": 2827, + "complete": True, + "server": {"host": "127.0.0.1", "port": 18878}, + "models": ["claude-haiku", "claude-opus"], +} + +LOG_RECORD_LINE = ( + '(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1")' + ' (port 18878))) (models ("claude-haiku" "claude-opus")))' +) + + +def test_a_record_is_written_on_one_line() -> None: + line = encode_line(LOG_RECORD) + assert "\n" not in line and "\r" not in line, line + assert line == LOG_RECORD_LINE + + +def test_a_line_is_valid_links_notation() -> None: + Parser().parse(encode_line(LOG_RECORD)) + + +def _ids(links: Any) -> list[str]: + """Every id in a parse tree, so a mangled parse can be recognised.""" + found: list[str] = [] + for link in links: + found.append(link.id or "") + found.extend(_ids(link.values)) + return found + + +def test_the_hand_rolled_dialect_is_not_read_as_a_record() -> None: + """The dialect a downstream project invented for the same need, which the + notation does not read back -- the reason this format exists. + + This parser does not raise on it; it swallows the parentheses into the data, + which loses the record just as surely. + """ + ids = _ids(Parser().parse('((:"bytes" 2827) (:"complete" true))')) + assert any("(" in name or ")" in name for name in ids), ids + + +@pytest.mark.parametrize( + "value", + [LOG_RECORD, [], {}, [{}, []], {"empty": []}, 42, None, "text"], +) +def test_both_forms_of_the_same_value_decode_alike(value: Any) -> None: + assert decode(encode_line(value)) == decode(encode(value)) + assert decode_line(encode_line(value)) == value + + +def test_a_string_keeps_its_own_characters_on_one_line() -> None: + value = {"text": 'quote " backslash \\ ünïcödé'} + line = encode_line(value) + assert line == "(o: (text 'quote \" backslash \\ ünïcödé'))" + assert decode_line(line) == value + + +def test_a_string_holding_a_newline_still_fits_on_one_line() -> None: + value = {"readable": "still visible", "multiline": "line1\nline2"} + line = encode_line(value) + assert line == '(o: (readable "still visible") (multiline (base64 "bGluZTEKbGluZTI=")))' + assert "\n" not in line, line + assert decode_line(line) == value + + +def test_a_one_pair_dict_is_not_a_two_element_list() -> None: + assert encode_line({"a": 1}) == "(o: (a 1))" + assert encode_line(["a", 1]) == '("a" 1)' + assert decode_line("(o: (a 1))") == {"a": 1} + assert decode_line('("a" 1)') == ["a", 1] + + +def test_the_empty_key_survives_the_round_trip() -> None: + value = {"": 2} + assert encode_line(value) == '(o: ("" 2))' + assert decode_line(encode_line(value)) == value + + +def test_a_marked_object_holding_something_that_is_not_a_pair_is_rejected() -> None: + with pytest.raises(ReadableFormatError, match="pairs"): + decode_line("(o: 1 2)") + + +def test_several_lines_are_not_one_record() -> None: + with pytest.raises(ReadableFormatError): + decode_line("(o: (a 1))\n(o: (b 2))") + + +def test_a_trailing_newline_is_not_a_second_record() -> None: + assert decode_line("(o: (a 1))\n") == {"a": 1} + + +def test_a_line_starting_with_none_is_still_read_as_a_line() -> None: + assert decode("(null 1)") == [None, 1] + assert decode("(o: (a null))") == {"a": None} + # The one document both forms claim: `(None)` is the compact null this + # language writes, and stays read that way, so documents written before this + # format keep decoding. + assert decode("(None)") is None From 7452d07490ff0ecd141f16083e5eecb3e8d833a6 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 27 Aug 2026 12:12:41 +0000 Subject: [PATCH 07/12] feat(csharp): write a record on one line with EncodeLine/DecodeLine --- csharp/src/Lino.Objects.Codec/ObjectCodec.cs | 84 ++++++- csharp/src/Lino.Objects.Codec/Readable.cs | 208 ++++++++++++++++- .../ReadableConformanceTests.cs | 59 ++++- .../SingleLineFormatTests.cs | 217 ++++++++++++++++++ 4 files changed, 550 insertions(+), 18 deletions(-) create mode 100644 csharp/tests/Lino.Objects.Codec.Tests/SingleLineFormatTests.cs diff --git a/csharp/src/Lino.Objects.Codec/ObjectCodec.cs b/csharp/src/Lino.Objects.Codec/ObjectCodec.cs index 24c7aca..56530ca 100644 --- a/csharp/src/Lino.Objects.Codec/ObjectCodec.cs +++ b/csharp/src/Lino.Objects.Codec/ObjectCodec.cs @@ -127,6 +127,26 @@ private void FindObjectsNeedingIds(object? obj, Dictionary? seen = /// String representation in readable Links Notation format public string Encode(object? obj, string indent) => Readable.Encode(obj, indent); + /// + /// Encode a C# object into the readable format on one line. + /// + /// + /// The result holds no newline, so an append-only log keeps one record per + /// line and stays greppable, tailable and countable by wc -l. See + /// for the shape. + /// + /// The C# object to encode + /// String representation in readable Links Notation format, on one line + public string EncodeLine(object? obj) => Readable.EncodeLine(obj); + + /// + /// Decode one line of a readable Links Notation log back into a C# object. + /// + /// The exact inverse of . + /// One line written by + /// Reconstructed C# object + public object? DecodeLine(string notation) => Readable.DecodeLine(notation); + /// /// Encode a C# object to the compact Links Notation format. /// @@ -654,6 +674,18 @@ private Link EncodeValue(object? obj, HashSet? visited = null, i "array", TypeList, "object", TypeDict, }; + /// + /// Markers a compact document writes without a payload. + /// + /// + /// (null) is a compact null, while (null 1) is a readable line + /// holding two values, so the marker alone does not decide the format. + /// + private static readonly HashSet EmptyBodyMarkers = new(StringComparer.Ordinal) + { + TypeNull, "None", + }; + /// /// Whether a document is in the compact format. /// @@ -677,27 +709,41 @@ public static bool IsCompactNotation(string notation) return false; } - var tokens = firstLine[1..] - .Split(new[] { ' ', '\t', '\r', '(', ')' }, StringSplitOptions.RemoveEmptyEntries); - - if (tokens.Length == 0) - { - return false; - } - - var marker = tokens[0]; + var (marker, rest) = SplitToken(firstLine[1..].TrimStart()); // Skip the `obj_N:` definition id, if present. if (marker.EndsWith(':')) { - if (!marker[..^1].StartsWith("obj_", StringComparison.Ordinal) || tokens.Length < 2) + if (!marker.StartsWith("obj_", StringComparison.Ordinal)) { return false; } - marker = tokens[1]; + (marker, rest) = SplitToken(rest.TrimStart()); + } + + if (!CompactTypeMarkers.Contains(marker)) + { + return false; } - return CompactTypeMarkers.Contains(marker); + // A compact `null` carries no payload, so `(null)` is a compact document + // while `(null 1)` is a readable line holding two values. + if (EmptyBodyMarkers.Contains(marker)) + { + return rest.TrimStart().StartsWith(')'); + } + + return true; + } + + /// + /// Split off the first token of a line: everything up to the next whitespace + /// or parenthesis, plus what follows it. + /// + private static (string Token, string Remainder) SplitToken(string input) + { + var end = input.IndexOfAny(new[] { ' ', '\t', '\r', '\n', '(', ')' }); + return end < 0 ? (input, string.Empty) : (input[..end], input[end..]); } } @@ -754,6 +800,20 @@ public static class Codec /// Reconstructed C# object public static object? Decode(string notation) => new ObjectCodec().Decode(notation); + /// + /// Encode an object into the readable format on one line. + /// + /// The C# object to encode + /// String representation in readable Links Notation format, on one line + public static string EncodeLine(object? obj) => new ObjectCodec().EncodeLine(obj); + + /// + /// Decode one line of a readable Links Notation log. + /// + /// One line written by + /// Reconstructed C# object + public static object? DecodeLine(string notation) => new ObjectCodec().DecodeLine(notation); + /// /// Decode the compact Links Notation format to a C# object. /// diff --git a/csharp/src/Lino.Objects.Codec/Readable.cs b/csharp/src/Lino.Objects.Codec/Readable.cs index 36f083d..01958c5 100644 --- a/csharp/src/Lino.Objects.Codec/Readable.cs +++ b/csharp/src/Lino.Objects.Codec/Readable.cs @@ -75,6 +75,30 @@ public CircularReferenceException(string message, Exception innerException) /// CRLF normalisation would corrupt) are marked individually as /// (base64 "…") instead of encoding the whole document. /// +/// +/// writes the same document on one line, so one record +/// is one line and an append-only log stays greppable, tailable and countable by +/// wc -l. Rows can no longer be told apart by line breaks there, so an +/// object names itself with the o link id the notation already has, and +/// its pairs are written as their own links: +/// +/// +/// (o: (type "RouterState") (server (o: (host "127.0.0.1") (port 18878))) (models ("claude-haiku" "claude-opus"))) +/// +/// +/// An object is (o: (key value) …) and an empty one is (o:); an +/// array is (value …) and an empty one is (); scalars are written +/// exactly as in the indented form. +/// +/// +/// The marker is what answers the ambiguity a flat layout otherwise has: without +/// it ((key value)) reads both as the one-pair object and as the array +/// holding the two-element array, and an empty key makes it worse. With it, a +/// bare ( ) is always an array and a marked one is always an object, so +/// every value — empty key included — survives the round trip. Consequently a +/// hand-written one-line link such as (a 1) is the two-element +/// array, not the one-pair object: on one line, objects say so. +/// /// public static class Readable { @@ -84,6 +108,9 @@ public static class Readable /// Marker used for values that cannot be represented as plain text. public const string Base64Marker = "base64"; + /// Link id naming an object in the single-line form, written as (o: …). + public const string ObjectMarker = "o"; + /// Characters that cannot appear in a bare (unquoted) reference. private static readonly char[] QuoteChars = { '"', '\'', '`' }; @@ -112,6 +139,44 @@ public static string Encode(object? value, string indent) /// The readable Links Notation document public static string Encode(object? value) => Encode(value, DefaultIndent); + /// + /// Encode a value into the readable, single-line Links Notation form. + /// + /// + /// The result never contains a newline, so one value is one line of an + /// append-only log. See the class documentation for the shape. + /// + /// The value to encode + /// The readable Links Notation document, on one line + /// If the value refers back to itself + public static string EncodeLine(object? value) + { + var output = new StringBuilder(); + WriteLineValue(value, output, new HashSet(ReferenceEqualityComparer.Instance)); + return output.ToString(); + } + + /// + /// Decode the readable, single-line Links Notation form back into a value. + /// + /// + /// This is the exact inverse of . Input spanning more + /// than one line is rejected: a line-based reader hands over one record at a + /// time, and silently accepting several would merge two records into one value. + /// + /// One line of a readable Links Notation log + /// The reconstructed value + /// If the document is not well formed or holds a line break + public static object? DecodeLine(string text) + { + var line = text.Trim('\n', '\r'); + if (line.Contains('\n') || line.Contains('\r')) + { + throw new FormatException("a single-line document cannot contain a line break"); + } + return Decode(line); + } + /// /// Decode the readable, indented Links Notation form back into a value. /// @@ -136,7 +201,7 @@ public static string Encode(object? value, string indent) return NodeToValue(rows[0][0]); } - return RowsToValue(rows, true); + return RowsToValue(rows, true, false); } // === Encoding === @@ -200,6 +265,58 @@ private static void WriteValue(object? value, string indent, int level, StringBu output.Append(FormatScalar(value)); } + /// + /// Write a value on one line. Objects name themselves with the o link + /// id and write each pair as its own link, so nothing depends on where lines + /// break. + /// + private static void WriteLineValue(object? value, StringBuilder output, HashSet path) + { + if (value is IDictionary dict) + { + EnterPath(dict, path); + if (dict.Count == 0) + { + // `()` is the empty array, so the empty object keeps its marker. + output.Append('(').Append(ObjectMarker).Append(":)"); + } + else + { + output.Append('(').Append(ObjectMarker).Append(':'); + foreach (var pair in dict) + { + output.Append(" (").Append(FormatKey(pair.Key)).Append(' '); + WriteLineValue(pair.Value, output, path); + output.Append(')'); + } + output.Append(')'); + } + path.Remove(dict); + return; + } + + if (value is System.Collections.IEnumerable items and not string) + { + EnterPath(items, path); + output.Append('('); + var first = true; + foreach (var item in items) + { + if (!first) + { + output.Append(' '); + } + first = false; + WriteLineValue(item, output, path); + } + output.Append(')'); + path.Remove(items); + return; + } + + output.Append(FormatScalar(value)); + } + /// /// Mark a container as being written, so a reference back to it is caught. /// @@ -349,6 +466,9 @@ private sealed class Node public bool Quoted { get; init; } public List> Rows { get; init; } = new(); public bool Multiline { get; init; } + + /// Whether the link named itself an object with the o: marker. + public bool IsObject { get; init; } } /// @@ -511,14 +631,35 @@ private Node ParseNode() if (token.Kind == TokenKind.Open) { Pos++; + var isObject = TakeObjectMarker(); var multiline = LinkIsMultiline(); var rows = ParseRows(false); - return new Node { IsRef = false, Rows = rows, Multiline = multiline }; + return new Node { IsRef = false, Rows = rows, Multiline = multiline, IsObject = isObject }; } throw new FormatException("unexpected token in readable notation"); } + /// + /// Consume the o: marker if the link that just opened carries one, + /// which is how the single-line form says "this link is an object, not an + /// array". + /// + private bool TakeObjectMarker() + { + if (Pos >= _tokens.Count) + { + return false; + } + var token = _tokens[Pos]; + if (token.Kind != TokenKind.Ref || token.Quoted || token.Value != ObjectMarker + ":") + { + return false; + } + Pos++; + return true; + } + /// /// Whether the link that just opened spans more than one line, which is /// what tells an empty object ((\n)) from an empty array (()). @@ -541,10 +682,17 @@ private bool LinkIsMultiline() } private static object? NodeToValue(Node node) => - node.IsRef ? RefToValue(node.Value, node.Quoted) : RowsToValue(node.Rows, node.Multiline); + node.IsRef + ? RefToValue(node.Value, node.Quoted) + : RowsToValue(node.Rows, node.Multiline, node.IsObject); - private static object? RowsToValue(List> rows, bool multiline) + private static object? RowsToValue(List> rows, bool multiline, bool objectMarker) { + if (objectMarker) + { + return MarkedObjectToValue(rows); + } + if (rows.Count == 0) { return multiline ? new Dictionary() : new List(); @@ -556,6 +704,22 @@ private bool LinkIsMultiline() return marked; } + // Written on one line, a link is a list of values: an object on one line + // says so with the `o:` marker, which is what keeps `(key value)` + // unambiguous. + if (!multiline) + { + var line = new List(); + foreach (var row in rows) + { + foreach (var node in row) + { + line.Add(NodeToValue(node)); + } + } + return line; + } + // `key value` on every line makes an object; anything else is a list of values. var isObject = rows.All(row => row.Count == 2 && row[0].IsRef); @@ -580,6 +744,42 @@ private bool LinkIsMultiline() return items; } + /// + /// Build the object a (o: (key value) …) link describes. Every value in + /// it is a pair, so anything else is a malformed document rather than a + /// silent array. + /// + private static object? MarkedObjectToValue(List> rows) + { + var result = new Dictionary(); + + foreach (var node in rows.SelectMany(row => row)) + { + if (node.IsRef || node.IsObject) + { + throw new FormatException( + $"an object marked '{ObjectMarker}:' holds (key value) pairs, " + + "found a value that is not a pair"); + } + if (node.Rows.Count != 1) + { + throw new FormatException( + $"an object marked '{ObjectMarker}:' holds (key value) pairs, " + + $"found a link of {node.Rows.Count.ToString(CultureInfo.InvariantCulture)} lines"); + } + var pair = node.Rows[0]; + if (pair.Count != 2 || !pair[0].IsRef) + { + throw new FormatException( + $"an object marked '{ObjectMarker}:' holds (key value) pairs, " + + $"found a link of {pair.Count.ToString(CultureInfo.InvariantCulture)} values"); + } + result[pair[0].Value] = NodeToValue(pair[1]); + } + + return result; + } + /// /// Recognise (base64 "…"), the individual marker for values that could /// not be written as text. A quoted base64 key is an ordinary object diff --git a/csharp/tests/Lino.Objects.Codec.Tests/ReadableConformanceTests.cs b/csharp/tests/Lino.Objects.Codec.Tests/ReadableConformanceTests.cs index d53ac6f..b278b13 100644 --- a/csharp/tests/Lino.Objects.Codec.Tests/ReadableConformanceTests.cs +++ b/csharp/tests/Lino.Objects.Codec.Tests/ReadableConformanceTests.cs @@ -1,10 +1,11 @@ -// Cross-language conformance tests for the readable, indented format. +// Cross-language conformance tests for the readable format, indented and on a +// single line. // // The fixtures in fixtures/readable-format/cases.json are shared by the // JavaScript, Python, Rust and C# suites. Each case is written by hand from the // format specification, so the four implementations check each other instead of // agreeing on a shared mistake: every language must encode `value` to exactly -// `text` and decode `text` back to exactly `value`. +// `text` and to exactly `line`, and decode both back to exactly `value`. using System.Globalization; using System.Text.Json; @@ -178,4 +179,58 @@ public void DecodesEachSharedTextBackToTheCaseValue(string name, JsonElement @ca var decoded = Codec.Decode(@case.GetProperty("text").GetString()!); Assert.True(Same(expected, decoded), $"case {name} decoded to a different value"); } + + [Theory] + [MemberData(nameof(AllCases))] + public void EncodesEachCaseToTheSharedLine(string name, JsonElement @case) + { + _ = name; + if (IsSkipped(@case)) + { + return; + } + var encoded = Codec.EncodeLine(Build(@case.GetProperty("value"))); + Assert.Equal(@case.GetProperty("line").GetString(), encoded); + } + + [Theory] + [MemberData(nameof(AllCases))] + public void DecodesEachSharedLineBackToTheCaseValue(string name, JsonElement @case) + { + if (IsSkipped(@case)) + { + return; + } + var expected = Build(@case.GetProperty("value")); + var decoded = Codec.DecodeLine(@case.GetProperty("line").GetString()!); + Assert.True(Same(expected, decoded), $"case {name} decoded to a different value"); + } + + /// A log record is one line, so no case may spread over two of them. + [Theory] + [MemberData(nameof(AllCases))] + public void NoSharedLineContainsALineBreak(string name, JsonElement @case) + { + var line = @case.GetProperty("line").GetString()!; + Assert.True( + !line.Contains('\n') && !line.Contains('\r'), + $"case {name} has a line break in its single-line form"); + } + + /// + /// reads both forms, so a log reader needs no flag + /// saying which one it holds. + /// + [Theory] + [MemberData(nameof(AllCases))] + public void ThePlainDecoderReadsEachSharedLine(string name, JsonElement @case) + { + if (IsSkipped(@case)) + { + return; + } + var expected = Build(@case.GetProperty("value")); + var decoded = Codec.Decode(@case.GetProperty("line").GetString()!); + Assert.True(Same(expected, decoded), $"case {name} decoded to a different value"); + } } diff --git a/csharp/tests/Lino.Objects.Codec.Tests/SingleLineFormatTests.cs b/csharp/tests/Lino.Objects.Codec.Tests/SingleLineFormatTests.cs new file mode 100644 index 0000000..d399344 --- /dev/null +++ b/csharp/tests/Lino.Objects.Codec.Tests/SingleLineFormatTests.cs @@ -0,0 +1,217 @@ +// Tests for the readable, single-line format produced by EncodeLine (issue #43). +// +// An append-only log wants one record per line: appending is one write, +// compaction cuts at a newline, and `grep`, `tail -f` and `wc -l` all treat a +// line as an event. Encode spreads a record over many lines and EncodeCompact +// hides it in base64, so neither serves that reader. + +using Xunit; +using Link.Foundation.Links.Notation; +using Lino.Objects.Codec; + +namespace Lino.Objects.Codec.Tests; + +/// +/// Checks the single-line form: one record, one line, read back exactly. +/// +public class SingleLineFormatTests +{ + /// A record of the shape an append-only log actually holds. + private static Dictionary LogRecord() => new() + { + ["bytes"] = 2827, + ["complete"] = true, + ["server"] = new Dictionary + { + ["host"] = "127.0.0.1", + ["port"] = 18878, + }, + ["models"] = new List { "claude-haiku", "claude-opus" }, + }; + + /// The dialect a downstream project invented for the same need. + private const string HandRolledDialect = "((:\"bytes\" 2827) (:\"complete\" true))"; + + [Fact] + public void ARecordIsWrittenOnOneLine() + { + var line = Codec.EncodeLine(LogRecord()); + Assert.DoesNotContain('\n', line); + Assert.DoesNotContain('\r', line); + Assert.Equal( + "(o: (bytes 2827) (complete true) (server (o: (host \"127.0.0.1\") (port 18878))) " + + "(models (\"claude-haiku\" \"claude-opus\")))", + line); + } + + [Fact] + public void ALineIsValidLinksNotation() + { + var line = Codec.EncodeLine(LogRecord()); + var links = new Parser().Parse(line); + Assert.NotNull(links); + Assert.NotEmpty(links); + } + + /// + /// The hand-rolled dialect is what this format replaces: it does not read + /// back as the record it was written from. + /// + [Fact] + public void TheHandRolledDialectIsNotReadAsARecord() + { + var decoded = Codec.Decode(HandRolledDialect); + Assert.False( + decoded is IDictionary dict + && dict.ContainsKey("bytes") + && dict.ContainsKey("complete"), + $"the hand-rolled dialect unexpectedly read back as a record: {decoded}"); + } + + [Fact] + public void BothFormsOfTheSameValueDecodeAlike() + { + var values = new object?[] + { + LogRecord(), + new List(), + new Dictionary(), + new List { new Dictionary(), new List() }, + new Dictionary { ["empty"] = new List() }, + 42, + null, + }; + + foreach (var value in values) + { + var fromLine = Codec.Decode(Codec.EncodeLine(value)); + var fromText = Codec.Decode(Codec.Encode(value)); + Assert.True(Equivalent(fromLine, fromText), $"the two forms disagree about {value}"); + Assert.True(Equivalent(Codec.DecodeLine(Codec.EncodeLine(value)), value)); + } + } + + [Fact] + public void AStringKeepsItsOwnCharactersOnOneLine() + { + var value = new Dictionary { ["text"] = "quote \" backslash \\ ünïcödé" }; + var line = Codec.EncodeLine(value); + Assert.Equal("(o: (text 'quote \" backslash \\ ünïcödé'))", line); + Assert.True(Equivalent(Codec.DecodeLine(line), value)); + } + + /// + /// A newline inside a string would end the record, so such a string is the + /// one thing written encoded -- individually, so the rest stays readable. + /// + [Fact] + public void AStringHoldingANewlineStillFitsOnOneLine() + { + var value = new Dictionary + { + ["readable"] = "still visible", + ["multiline"] = "line1\nline2", + }; + var line = Codec.EncodeLine(value); + Assert.Equal( + "(o: (readable \"still visible\") (multiline (base64 \"bGluZTEKbGluZTI=\")))", + line); + Assert.DoesNotContain('\n', line); + Assert.True(Equivalent(Codec.DecodeLine(line), value)); + } + + /// + /// The one ambiguity a flat layout has: is (a 1) a one-pair object or + /// a two-element array? On one line an object says so with the o: + /// marker, so both values keep their own spelling. + /// + [Fact] + public void AOnePairObjectIsNotATwoElementArray() + { + var @object = new Dictionary { ["a"] = 1 }; + var array = new List { "a", 1 }; + + Assert.Equal("(o: (a 1))", Codec.EncodeLine(@object)); + Assert.Equal("(\"a\" 1)", Codec.EncodeLine(array)); + Assert.True(Equivalent(Codec.DecodeLine("(o: (a 1))"), @object)); + Assert.True(Equivalent(Codec.DecodeLine("(\"a\" 1)"), array)); + } + + /// + /// Because the marker answers it, the empty key round-trips instead of being + /// rejected: ("" 2) is a pair like any other inside a marked object. + /// + [Fact] + public void TheEmptyKeySurvivesTheRoundTrip() + { + var value = new Dictionary { [""] = 2 }; + Assert.Equal("(o: (\"\" 2))", Codec.EncodeLine(value)); + Assert.True(Equivalent(Codec.DecodeLine(Codec.EncodeLine(value)), value)); + } + + [Fact] + public void AMarkedObjectHoldingSomethingThatIsNotAPairIsRejected() + { + var error = Assert.Throws(() => Codec.DecodeLine("(o: 1 2)")); + Assert.Contains("pairs", error.Message, StringComparison.Ordinal); + } + + /// + /// Reading a log means handing over one record at a time, so a decoder that + /// silently accepted two lines would merge two records into one value. + /// + [Fact] + public void SeveralLinesAreNotOneRecord() + { + Assert.Throws(() => Codec.DecodeLine("(o: (a 1))\n(o: (b 2))")); + } + + /// + /// A trailing newline is what a line reader may keep, so it is trimmed rather + /// than refused. + /// + [Fact] + public void ATrailingNewlineIsNotASecondRecord() + { + var decoded = Codec.DecodeLine("(o: (a 1))\n"); + Assert.True(Equivalent(decoded, new Dictionary { ["a"] = 1 })); + } + + /// + /// A line whose first value is null is a readable line, not the compact null: + /// Decode must not route it to the base64 reader. + /// + [Fact] + public void ALineStartingWithNullIsStillReadAsALine() + { + Assert.True(Equivalent(Codec.Decode("(null 1)"), new List { null, 1 })); + Assert.True(Equivalent( + Codec.Decode("(o: (a null))"), + new Dictionary { ["a"] = null })); + // The one document both forms claim: `(null)` is the compact null, and + // stays read that way, so documents written before this format keep + // decoding. + Assert.Null(Codec.Decode("(null)")); + } + + /// Structural comparison, since dictionaries and lists compare by reference. + private static bool Equivalent(object? left, object? right) + { + switch (left, right) + { + case (null, null): + return true; + case (IDictionary a, IDictionary b): + return a.Count == b.Count + && a.Keys.SequenceEqual(b.Keys) + && a.All(entry => Equivalent(entry.Value, b[entry.Key])); + case (System.Collections.IEnumerable a and not string, System.Collections.IEnumerable b and not string): + var left_items = a.Cast().ToList(); + var right_items = b.Cast().ToList(); + return left_items.Count == right_items.Count + && left_items.Zip(right_items).All(pair => Equivalent(pair.First, pair.Second)); + default: + return Equals(left, right); + } + } +} From fdaac663c7fc9df115d6647d5e30dd99b21ee7b0 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 27 Aug 2026 12:13:22 +0000 Subject: [PATCH 08/12] chore: ignore cargo build output under experiments/ --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index af52c44..45f5a72 100644 --- a/.gitignore +++ b/.gitignore @@ -172,3 +172,6 @@ htmlcov/ .dmypy.json dmypy.json .ruff_cache/ + +# Cargo build output of the scratch crates under experiments/ +experiments/**/target/ From fa182b17cdc3e3e2da24767313969de7bfb88f2f Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 27 Aug 2026 12:14:55 +0000 Subject: [PATCH 09/12] docs: describe the single-line form in every readme --- README.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++- csharp/README.md | 49 ++++++++++++++++++++++++++++++++++++++++++ js/README.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++ python/README.md | 51 ++++++++++++++++++++++++++++++++++++++++++++ rust/README.md | 27 ++++++++++++++++++++++-- 5 files changed, 232 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4ba2830..c299941 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ All implementations share the same design philosophy and provide feature parity. - **C#**: `null`, `bool`, `int`, `long`, `float`, `double`, `string`, `List`, `Dictionary` - Special float/number values: `NaN`, `Infinity`, `-Infinity` - **Readable by Default**: In every language `encode()` writes indented, plain-text Links Notation; the previous single-line base64 form stays available as `encode_compact()` (alias `encode_obfuscated()`) +- **One Record per Line**: `encode_line()` writes the same readable document on one line and `decode_line()` reads it back exactly, so an append-only log stays greppable, tailable and countable by `wc -l` - **Object Identity**: Shared references and circular references are preserved by the compact format via object ids; the readable format is a plain tree and raises a circular-reference error instead - **Full Unicode**: Strings are written as text; only a value that cannot be written as text (one holding control characters) is base64-encoded, and it is marked individually as `(base64 "…")` - **Opt-in Tracing**: Set `LINO_CODEC_DEBUG=1` to trace encoding and decoding, the same way in every language @@ -115,8 +116,37 @@ assert_eq!(decoded, data); ) ``` +For an append-only log, `encode_line()` writes the same document on one line: + +```lino +(o: (name "Alice") (age 30) (active true)) +``` + +```python +from link_notation_objects_codec import encode_line, decode_line + +decode_line(encode_line(data)) == data +``` + +```javascript +import { encodeLine, decodeLine } from "lino-objects-codec"; + +decodeLine({ notation: encodeLine({ obj: data }) }); +``` + +```rust +use lino_objects_codec::{decode_line, encode_line}; + +assert_eq!(decode_line(&encode_line(&data)).unwrap(), data); +``` + +```csharp +var line = Codec.EncodeLine(data); +var record = Codec.DecodeLine(line); +``` + The single-line base64 form is still available as `encode_compact()` (alias -`encode_obfuscated()`) in every language, and `decode()` accepts both forms. +`encode_obfuscated()`) in every language, and `decode()` accepts all three forms. ### C# @@ -395,6 +425,29 @@ object, bare-value lines make an array: - The four languages produce byte-identical output, checked by the shared fixtures in [`fixtures/readable-format/cases.json`](fixtures/readable-format/cases.json) +### Single-line format (`encode_line`) + +The same readable document written on one line, so an append-only log holds one +record per line — appending is one write, compaction cuts at a newline, and +`grep`, `tail -f` and `wc -l` all treat a line as one event: + +```lino +(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1") (port 18878)))) +``` + +- An object is `(o: (key value) …)` and an empty object is `(o:)` +- An array is `(value …)` and an empty array is `()` +- Scalars and strings are written exactly as in the indented form, so a string + keeps its own characters and a number keeps its type +- The `o` marker is what removes the ambiguity a flat layout otherwise has: + without it `((key value))` reads both as a one-pair object and as an array + holding a two-element array. With it a bare `( )` on one line is always an + array, so a *hand-written* `(a 1)` is the two-element array — on one line, + objects say so +- `decode()` reads this form too, so a log reader needs no flag saying which form + a file holds; `decode_line()` is the exact inverse of `encode_line()` and + rejects input spanning more than one line + ### Compact format (`encode_compact`) The previous single-line form, kept for compatibility and for the object graphs diff --git a/csharp/README.md b/csharp/README.md index c4aae7a..1c007d2 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -15,6 +15,7 @@ A C# library for working with Links Notation format. This library provides unive - Collections: `List`, `Dictionary` - Special float values: `NaN`, `Infinity`, `-Infinity` - **Readable by Default**: `Codec.Encode()` writes plain, indented text that can be read and reviewed +- **One Record per Line**: `Codec.EncodeLine()` writes the same document on one line and `Codec.DecodeLine()` reads it back exactly, so an append-only log stays greppable, tailable and countable by `wc -l` - **Object Identity**: Shared references and circular references are preserved by the compact format (`Codec.EncodeCompact`) via object ids - **Full Unicode**: Strings are written as text; only a value that cannot be written as text (one holding control characters) is base64-encoded, and it is marked individually as `(base64 "…")` - **Opt-in Tracing**: Set `LINO_CODEC_DEBUG=1` to trace encoding and decoding, the same way in every language @@ -153,6 +154,7 @@ decoded = Codec.Decode(Codec.Encode(complexData)); | --- | --- | | `Codec.Encode(obj)` | Readable, indented Links Notation (the default) | | `Codec.Encode(obj, "\t")` | Same, with a custom indentation string | +| `Codec.EncodeLine(obj)` | The same readable document on one line, for append-only logs | | `Codec.EncodeCompact(obj)` | The previous single-line, base64 form | | `Codec.EncodeObfuscated(obj)` | Alias of `Codec.EncodeCompact` | @@ -217,6 +219,25 @@ bare-value lines make a list: base64-encoded on its own and marked as `(base64 "bGluZTEKbGluZTI=")`; everything around it stays readable +### Single-line format (`Codec.EncodeLine`) + +The same readable document on one line, so an append-only log holds one record +per line -- appending is one write, compaction cuts at a newline, and `grep`, +`tail -f` and `wc -l` all treat a line as one event: + +```lino +(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1") (port 18878)))) +``` + +- A dictionary is `(o: (key value) …)` and an empty dictionary is `(o:)` +- A list is `(value …)` and an empty list is `()` +- Scalars and strings are written exactly as in the indented form +- The `o` marker removes the ambiguity a flat layout otherwise has: a bare `( )` + on one line is always a list, so a *hand-written* `(a 1)` is the two-element + list, not the one-pair dictionary +- `Codec.Decode` reads this form too; `Codec.DecodeLine` is its exact inverse and + rejects input spanning more than one line + ### Compact format (`Codec.EncodeCompact`) The previous single-line form, kept for compatibility and for the object graphs @@ -282,6 +303,34 @@ Decode Links Notation format to a C# object. **Throws:** - `InvalidOperationException` - If the type marker is unknown +#### `Codec.EncodeLine(object? obj)` + +Encode a C# object into the readable format on one line. + +**Parameters:** +- `obj` - The C# object to encode (can be null) + +**Returns:** +- String representation in readable Links Notation format, holding no newline + +```csharp +Codec.EncodeLine(new Dictionary { ["age"] = 30 }); // (o: (age 30)) +``` + +#### `Codec.DecodeLine(string notation)` + +Decode one line of a readable Links Notation log. The exact inverse of +`Codec.EncodeLine`. + +**Parameters:** +- `notation` - One line written by `Codec.EncodeLine` + +**Returns:** +- Reconstructed C# object (or null) + +**Throws:** +- `FormatException` - If the input spans more than one line or is malformed + ### ObjectCodec Class The main codec class that performs encoding and decoding. The static `Codec` class creates a new instance for each operation to ensure thread safety. diff --git a/js/README.md b/js/README.md index 62a653d..acc4c62 100644 --- a/js/README.md +++ b/js/README.md @@ -27,6 +27,7 @@ These tools enable easy implementation of higher-level features like: - Collections: `Array`, `Object` - Special number values: `NaN`, `Infinity`, `-Infinity` - **Readable by Default**: `encode({ obj })` writes plain, indented text that can be read and reviewed +- **One Record per Line**: `encodeLine({ obj })` writes the same document on one line and `decodeLine({ notation })` reads it back exactly, so an append-only log stays greppable, tailable and countable by `wc -l` - **Object Identity**: Shared references and circular references are preserved by the compact format (`encodeCompact`) via object ids - **Full Unicode**: Strings are written as text; only a value that cannot be written as text (one holding control characters) is base64-encoded, and it is marked individually as `(base64 "…")` - **Opt-in Tracing**: Set `LINO_CODEC_DEBUG=1` to trace encoding and decoding, the same way in every language @@ -112,6 +113,7 @@ console.log(decoded.self === decoded); | ------------------------------- | ----------------------------------------------- | | `encode({ obj })` | Readable, indented Links Notation (the default) | | `encode({ obj, indent: '\t' })` | Same, with a custom indentation string | +| `encodeLine({ obj })` | The same document on one line, for append-only logs | | `encodeCompact({ obj })` | The previous single-line, base64 form | | `encodeObfuscated({ obj })` | Alias of `encodeCompact` | @@ -334,6 +336,25 @@ bare-value lines make an array: base64-encoded on its own and marked as `(base64 "bGluZTEKbGluZTI=")`; everything around it stays readable +### Single-line format (`encodeLine`) + +The same readable document on one line, so an append-only log holds one record +per line — appending is one write, compaction cuts at a newline, and `grep`, +`tail -f` and `wc -l` all treat a line as one event: + +```lino +(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1") (port 18878)))) +``` + +- An object is `(o: (key value) …)` and an empty object is `(o:)` +- An array is `(value …)` and an empty array is `()` +- Scalars and strings are written exactly as in the indented form +- The `o` marker removes the ambiguity a flat layout otherwise has: a bare `( )` + on one line is always an array, so a _hand-written_ `(a 1)` is the two-element + array, not the one-pair object +- `decode({ notation })` reads this form too; `decodeLine({ notation })` is its + exact inverse and rejects input spanning more than one line + ### Compact format (`encodeCompact`) The previous single-line form, kept for compatibility and for the object graphs @@ -441,6 +462,38 @@ Decode Links Notation format to a JavaScript object. - Reconstructed JavaScript object +#### `encodeLine({ obj: obj })` + +Encode a JavaScript object into the readable format on one line. + +**Parameters:** + +- `options.obj` - The JavaScript object to encode + +**Returns:** + +- String representation in readable Links Notation format, holding no newline + +```javascript +encodeLine({ obj: { age: 30 } }); // '(o: (age 30))' +``` + +#### `decodeLine({ notation: notation })` + +Decode one line of a readable Links Notation log. The exact inverse of `encodeLine`. + +**Parameters:** + +- `options.notation` - One line written by `encodeLine` + +**Returns:** + +- Reconstructed JavaScript object + +**Throws:** + +- `SyntaxError` - If the input spans more than one line or is malformed + #### `ObjectCodec` The main codec class that performs encoding and decoding. The module-level `encode({ obj: )` and `decode({ notation: } })` functions use a shared instance of this class. diff --git a/python/README.md b/python/README.md index f4646c3..2539018 100644 --- a/python/README.md +++ b/python/README.md @@ -10,6 +10,7 @@ A Python library to encode/decode objects to/from Links Notation format. This li ## Features - **Readable by Default**: `encode()` writes plain, indented text that can be read and reviewed +- **One Record per Line**: `encode_line()` writes the same document on one line and `decode_line()` reads it back exactly, so an append-only log stays greppable, tailable and countable by `wc -l` - **Universal Serialization**: Encode Python objects to Links Notation format - **Type Support**: Handle all common Python types: - Basic types: `None`, `bool`, `int`, `float`, `str` @@ -120,6 +121,7 @@ assert decode(encode(complex_data)) == complex_data | --- | --- | | `encode(obj)` | Readable, indented Links Notation (the default) | | `encode(obj, indent="\t")` | Same, with a custom indentation string | +| `encode_line(obj)` | The same readable document on one line, for append-only logs | | `encode_compact(obj)` | The previous single-line, base64 form | | `encode_obfuscated(obj)` | Alias of `encode_compact` | @@ -192,6 +194,25 @@ list: base64-encoded on its own and marked as `(base64 "bGluZTEKbGluZTI=")`; everything around it stays readable +### Single-line format (`encode_line`) + +The same readable document on one line, so an append-only log holds one record +per line — appending is one write, compaction cuts at a newline, and `grep`, +`tail -f` and `wc -l` all treat a line as one event: + +```lino +(o: (bytes 2827) (complete true) (server (o: (host "127.0.0.1") (port 18878)))) +``` + +- A dict is `(o: (key value) …)` and an empty dict is `(o:)` +- A list is `(value …)` and an empty list is `()` +- Scalars and strings are written exactly as in the indented form +- The `o` marker removes the ambiguity a flat layout otherwise has: a bare `( )` + on one line is always a list, so a *hand-written* `(a 1)` is the two-element + list, not the one-pair dict +- `decode()` reads this form too; `decode_line()` is its exact inverse and + rejects input spanning more than one line + ### Compact format (`encode_compact`) The previous single-line form, kept for compatibility and for the object graphs @@ -253,6 +274,36 @@ Decode Links Notation format to a Python object. **Returns:** - Reconstructed Python object +### `encode_line(obj: Any) -> str` + +Encode a Python object into the readable format on one line. + +**Parameters:** +- `obj`: The Python object to encode + +**Returns:** +- String representation in readable Links Notation format, holding no newline + +```python +>>> from link_notation_objects_codec import encode_line +>>> encode_line({"age": 30}) +'(o: (age 30))' +``` + +### `decode_line(notation: str) -> Any` + +Decode one line of a readable Links Notation log. The exact inverse of +`encode_line()`. + +**Parameters:** +- `notation`: One line written by `encode_line()` + +**Returns:** +- Reconstructed Python object + +**Raises:** +- `ReadableFormatError`: If the input spans more than one line or is malformed + ### `ObjectCodec` The main codec class that performs encoding and decoding. The module-level `encode()` and `decode()` functions use a shared instance of this class. diff --git a/rust/README.md b/rust/README.md index 82eae66..f45abf9 100644 --- a/rust/README.md +++ b/rust/README.md @@ -32,6 +32,7 @@ lino-objects-codec = "0.1" - **Object Identity**: Shared references are preserved by the compact format - **Opt-in Tracing**: Set `LINO_CODEC_DEBUG=1` to trace encoding and decoding, the same way in every language - **Readable by Default**: `encode()` writes indented, plain-text Links Notation; keys and values stay legible and diffable +- **One Record per Line**: `encode_line()` writes the same document on one line and `decode_line()` reads it back exactly, so an append-only log stays greppable, tailable and countable by `wc -l` - **UTF-8 Support**: Full Unicode string support written as text; only values that cannot be written as text (control characters) are base64-encoded, and each is marked individually - **Simple API**: Easy-to-use `encode()` and `decode()` functions @@ -72,6 +73,7 @@ The encoded document reads as: | --- | --- | | `encode(value)` | Readable, indented Links Notation (the default) | | `encode_with_indent(value, "\t")` | Same, with a custom indentation string | +| `encode_line(value)` | The same readable document on one line, for append-only logs | | `encode_compact(value)` | The previous single-line base64 form | | `encode_obfuscated(value)` | Alias of `encode_compact` | @@ -133,6 +135,27 @@ assert_eq!(encoded, "42"); Same as `encode()`, but with a custom indentation string (the default is two spaces). +#### `encode_line(value: &LinoValue) -> String` + +Encode a value to the readable form on one line, so one record is one line of an +append-only log. An object names itself with the `o` marker, which is what keeps +`(key value)` unambiguous; a bare link on one line is always an array. + +```rust +let value = LinoValue::object([("age", LinoValue::Int(30))]); +assert_eq!(encode_line(&value), "(o: (age 30))"); +``` + +#### `decode_line(notation: &str) -> Result` + +The exact inverse of `encode_line()`. Input spanning more than one line is +rejected, so two log records can never be merged into one value. + +```rust +let decoded = decode_line("(o: (age 30))").unwrap(); +assert_eq!(decoded, LinoValue::object([("age", LinoValue::Int(30))])); +``` + #### `encode_compact(value: &LinoValue) -> String` Encode a value to the single-line, base64 form used before version 0.3. @@ -148,8 +171,8 @@ Alias of `encode_compact()`, named after what the base64 form actually does to t #### `decode(notation: &str) -> Result` -Decode Links Notation format to a value. Both the readable and the compact form -are accepted. +Decode Links Notation format to a value. The readable form, the single-line form +and the compact form are all accepted. ```rust let decoded = decode("42").unwrap(); From 6e16521b3422efad6709d0e9ba837d6b56345141 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 27 Aug 2026 12:19:34 +0000 Subject: [PATCH 10/12] docs(examples): show an append-only log with one record per line --- .github/workflows/rust.yml | 6 ++-- csharp/examples/BasicUsage.cs | 35 ++++++++++++++++++++++ js/README.md | 12 ++++---- js/examples/append_only_log.js | 42 ++++++++++++++++++++++++++ python/examples/append_only_log.py | 44 ++++++++++++++++++++++++++++ rust/examples/append_only_log.rs | 47 ++++++++++++++++++++++++++++++ 6 files changed, 178 insertions(+), 8 deletions(-) create mode 100644 js/examples/append_only_log.js create mode 100644 python/examples/append_only_log.py create mode 100644 rust/examples/append_only_log.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index afa1a4b..49837e7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -286,9 +286,11 @@ jobs: working-directory: ./rust run: cargo test --doc --verbose - - name: Run example + - name: Run examples working-directory: ./rust - run: cargo run --example basic_usage + run: | + cargo run --example basic_usage + cargo run --example append_only_log # === BUILD === # Build package - only runs if lint and test pass diff --git a/csharp/examples/BasicUsage.cs b/csharp/examples/BasicUsage.cs index 164e906..4168dbe 100644 --- a/csharp/examples/BasicUsage.cs +++ b/csharp/examples/BasicUsage.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Linq; using Lino.Objects.Codec; Console.WriteLine("=== lino-objects-codec C# Basic Usage Example ===\n"); @@ -146,4 +147,38 @@ Console.WriteLine(); +// An append-only log wants one record per line: appending is one write, a +// compactor can cut the file at any newline, and `grep`, `tail -f` and `wc -l` +// all treat one line as one event. +Console.WriteLine("Append-only log, one record per line:"); + +Dictionary Record(string phase, int bytes, bool complete) => new() +{ + { "phase", phase }, + { "bytes", bytes }, + { "complete", complete } +}; + +var entries = new[] +{ + Record("stream_start", 0, false), + Record("stream_chunk", 1024, false), + Record("stream_end", 2827, true) +}; +var log = string.Concat(entries.Select(entry => Codec.EncodeLine(entry) + "\n")); +Console.Write(log); + +var lines = log.TrimEnd('\n').Split('\n'); +Console.WriteLine($" records: {lines.Length}"); + +// Reading: a line reader hands over one record at a time. +var lastRecord = Codec.DecodeLine(lines[^1]) as Dictionary; +Console.WriteLine($" last record phase: {lastRecord?["phase"]}"); + +// Filtering: the text stays readable, so plain string tools still work. +var finished = lines.Count(line => line.Contains("(complete true)", StringComparison.Ordinal)); +Console.WriteLine($" finished records: {finished}"); + +Console.WriteLine(); + Console.WriteLine("=== Example completed successfully! ==="); diff --git a/js/README.md b/js/README.md index acc4c62..16f238b 100644 --- a/js/README.md +++ b/js/README.md @@ -109,13 +109,13 @@ console.log(decoded.self === decoded); ## Output Formats -| Function | Output | -| ------------------------------- | ----------------------------------------------- | -| `encode({ obj })` | Readable, indented Links Notation (the default) | -| `encode({ obj, indent: '\t' })` | Same, with a custom indentation string | +| Function | Output | +| ------------------------------- | --------------------------------------------------- | +| `encode({ obj })` | Readable, indented Links Notation (the default) | +| `encode({ obj, indent: '\t' })` | Same, with a custom indentation string | | `encodeLine({ obj })` | The same document on one line, for append-only logs | -| `encodeCompact({ obj })` | The previous single-line, base64 form | -| `encodeObfuscated({ obj })` | Alias of `encodeCompact` | +| `encodeCompact({ obj })` | The previous single-line, base64 form | +| `encodeObfuscated({ obj })` | Alias of `encodeCompact` | `decode({ notation })` accepts every one of them, so files written by older versions keep working and are rewritten in the readable form the next time they diff --git a/js/examples/append_only_log.js b/js/examples/append_only_log.js new file mode 100644 index 0000000..5a17ad5 --- /dev/null +++ b/js/examples/append_only_log.js @@ -0,0 +1,42 @@ +/** + * Writing an append-only log with one record per line (issue #43). + * + * `encodeLine` keeps a record on one line, so appending is one write, a + * compactor can cut the file at any newline, and `grep`, `tail -f` and `wc -l` + * all treat one line as one event. `decodeLine` reads a line back exactly. + */ + +import assert from 'node:assert'; +import { encodeLine, decodeLine } from '../src/index.js'; + +const record = (phase, bytes, complete) => ({ phase, bytes, complete }); + +function main() { + console.log('=== Append-only log, one record per line ===\n'); + + // Appending: each record becomes exactly one line of the file. + const entries = [ + record('stream_start', 0, false), + record('stream_chunk', 1024, false), + record('stream_end', 2827, true), + ]; + const log = entries + .map((entry) => `${encodeLine({ obj: entry })}\n`) + .join(''); + process.stdout.write(log); + + // Counting: one line is one event, so `wc -l` answers how many there were. + const lines = log.trimEnd().split('\n'); + console.log(`\nrecords: ${lines.length}`); + + // Reading: a line reader hands over one record at a time. + const decoded = decodeLine({ notation: lines[lines.length - 1] }); + console.log(`last record: ${JSON.stringify(decoded)}`); + assert.deepStrictEqual(decoded, record('stream_end', 2827, true)); + + // Filtering: the text stays readable, so plain string tools still work. + const finished = lines.filter((line) => line.includes('(complete true)')); + console.log(`finished records: ${finished.length}`); +} + +main(); diff --git a/python/examples/append_only_log.py b/python/examples/append_only_log.py new file mode 100644 index 0000000..4955546 --- /dev/null +++ b/python/examples/append_only_log.py @@ -0,0 +1,44 @@ +"""Writing an append-only log with one record per line (issue #43). + +``encode_line`` keeps a record on one line, so appending is one write, a +compactor can cut the file at any newline, and ``grep``, ``tail -f`` and +``wc -l`` all treat one line as one event. ``decode_line`` reads a line back +exactly. +""" + +from link_notation_objects_codec import decode_line, encode_line + + +def record(phase: str, num_bytes: int, complete: bool) -> dict: + """Build a record of the shape an append-only log actually holds.""" + return {"phase": phase, "bytes": num_bytes, "complete": complete} + + +def main() -> None: + print("=== Append-only log, one record per line ===\n") + + # Appending: each record becomes exactly one line of the file. + entries = [ + record("stream_start", 0, False), + record("stream_chunk", 1024, False), + record("stream_end", 2827, True), + ] + log = "".join(f"{encode_line(entry)}\n" for entry in entries) + print(log, end="") + + # Counting: one line is one event, so `wc -l` answers how many there were. + lines = log.splitlines() + print(f"\nrecords: {len(lines)}") + + # Reading: a line reader hands over one record at a time. + decoded = decode_line(lines[-1]) + print(f"last record: {decoded}") + assert decoded == record("stream_end", 2827, True) + + # Filtering: the text stays readable, so plain string tools still work. + finished = [line for line in lines if "(complete true)" in line] + print(f"finished records: {len(finished)}") + + +if __name__ == "__main__": + main() diff --git a/rust/examples/append_only_log.rs b/rust/examples/append_only_log.rs new file mode 100644 index 0000000..a63a3c9 --- /dev/null +++ b/rust/examples/append_only_log.rs @@ -0,0 +1,47 @@ +//! Writing an append-only log with one record per line (issue #43). +//! +//! `encode_line` keeps a record on one line, so appending is one write, a +//! compactor can cut the file at any newline, and `grep`, `tail -f` and `wc -l` +//! all treat one line as one event. `decode_line` reads a line back exactly. + +use lino_objects_codec::{decode_line, encode_line, LinoValue}; +use std::fmt::Write as _; + +fn record(phase: &str, bytes: i64, complete: bool) -> LinoValue { + LinoValue::object([ + ("phase", LinoValue::String(phase.to_string())), + ("bytes", LinoValue::Int(bytes)), + ("complete", LinoValue::Bool(complete)), + ]) +} + +fn main() { + println!("=== Append-only log, one record per line ===\n"); + + // Appending: each record becomes exactly one line of the file. + let mut log = String::new(); + for entry in [ + record("stream_start", 0, false), + record("stream_chunk", 1024, false), + record("stream_end", 2827, true), + ] { + writeln!(log, "{}", encode_line(&entry)).expect("writing to a string cannot fail"); + } + print!("{log}"); + + // Counting: one line is one event, so `wc -l` answers how many there were. + println!("\nrecords: {}", log.lines().count()); + + // Reading: a line reader hands over one record at a time. + let last = log.lines().next_back().expect("the log holds records"); + let decoded = decode_line(last).expect("a line written by encode_line reads back"); + println!("last record: {decoded:?}"); + assert_eq!(decoded, record("stream_end", 2827, true)); + + // Filtering: the text stays readable, so plain string tools still work. + let finished = log + .lines() + .filter(|line| line.contains("(complete true)")) + .count(); + println!("finished records: {finished}"); +} From 50296203991102b6bee4ccda5bccc5387b20fe53 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 27 Aug 2026 12:21:01 +0000 Subject: [PATCH 11/12] chore(release): add changelog fragments for the single-line format --- ...0827_090000_issue_43_single_line_format.md | 23 +++++++++++++++++ ...0827_090000_issue_43_single_line_format.md | 23 +++++++++++++++++ ...0827_090000_issue_43_single_line_format.md | 24 ++++++++++++++++++ ...0827_090000_issue_43_single_line_format.md | 25 +++++++++++++++++++ 4 files changed, 95 insertions(+) create mode 100644 csharp/.changeset/20260827_090000_issue_43_single_line_format.md create mode 100644 js/.changeset/20260827_090000_issue_43_single_line_format.md create mode 100644 python/changelog.d/20260827_090000_issue_43_single_line_format.md create mode 100644 rust/changelog.d/20260827_090000_issue_43_single_line_format.md diff --git a/csharp/.changeset/20260827_090000_issue_43_single_line_format.md b/csharp/.changeset/20260827_090000_issue_43_single_line_format.md new file mode 100644 index 0000000..72975d9 --- /dev/null +++ b/csharp/.changeset/20260827_090000_issue_43_single_line_format.md @@ -0,0 +1,23 @@ +--- +'Lino.Objects.Codec': minor +--- + +Add `Codec.EncodeLine` and `Codec.DecodeLine` (and the matching `ObjectCodec` +methods): the readable format written on one line, so an append-only log holds +one record per line. Appending is one write, compaction cuts at a newline, and +`grep`, `tail -f` and `wc -l` treat a line as one event. The output is valid +Links Notation, keeps numbers, booleans and `null` bare so types survive the +round trip, and `Decode(EncodeLine(v))` equals `Decode(Encode(v))`. + +`Readable.ObjectMarker` (`o`) tells an object from an array on one line: +`(o: (bytes 2827) (complete true))` is a record, `("a" 1)` is a two-element +array, `(o:)` is the empty object and `()` the empty array. Because the marker +is part of the notation, the empty key round-trips as `(o: ("" 2))`. The +single-line spelling of every shared fixture is pinned in +`fixtures/readable-format/cases.json`, so all four languages write the same +bytes. + +Also fixes `IsCompactNotation`, which used to claim a readable single-line +document such as `(null 1)`; the document `(null)` stays the compact null so +older documents keep decoding. See +[issue #43](https://github.com/link-foundation/lino-objects-codec/issues/43). diff --git a/js/.changeset/20260827_090000_issue_43_single_line_format.md b/js/.changeset/20260827_090000_issue_43_single_line_format.md new file mode 100644 index 0000000..319f0ce --- /dev/null +++ b/js/.changeset/20260827_090000_issue_43_single_line_format.md @@ -0,0 +1,23 @@ +--- +'lino-objects-codec': minor +--- + +Add `encodeLine` and `decodeLine`: the readable format written on one line, so +an append-only log holds one record per line. Appending is one write, compaction +cuts at a newline, and `grep`, `tail -f` and `wc -l` treat a line as one event. +The output is valid Links Notation, keeps numbers, booleans and `null` bare so +types survive the round trip, and `decode(encodeLine(v))` equals +`decode(encode(v))`. + +The exported `OBJECT_MARKER` (`o`) tells an object from an array on one line: +`(o: (bytes 2827) (complete true))` is a record, `("a" 1)` is a two-element +array, `(o:)` is the empty object and `()` the empty array. Because the marker +is part of the notation, the empty key round-trips as `(o: ("" 2))`. The +single-line spelling of every shared fixture is pinned in +`fixtures/readable-format/cases.json`, so all four languages write the same +bytes. + +Also fixes `decode`, which used to route a readable single-line document such as +`(null 1)` to the compact (base64) reader; the document `(null)` stays the +compact null so older documents keep decoding. See +[issue #43](https://github.com/link-foundation/lino-objects-codec/issues/43). diff --git a/python/changelog.d/20260827_090000_issue_43_single_line_format.md b/python/changelog.d/20260827_090000_issue_43_single_line_format.md new file mode 100644 index 0000000..33d769f --- /dev/null +++ b/python/changelog.d/20260827_090000_issue_43_single_line_format.md @@ -0,0 +1,24 @@ +### Added + +- `encode_line` and `decode_line`: the readable format written on one line, so + an append-only log holds one record per line. Appending is one write, + compaction cuts at a newline, and `grep`, `tail -f` and `wc -l` treat a line + as one event. The output is valid Links Notation, keeps numbers, booleans and + `None` bare so types survive the round trip, and `decode(encode_line(v))` + equals `decode(encode(v))`. See + [issue #43](https://github.com/link-foundation/lino-objects-codec/issues/43). +- `OBJECT_MARKER`, the `o` link id that tells an object from an array on one + line: `(o: (bytes 2827) (complete true))` is a record, `("a" 1)` is a + two-element array, `(o:)` is the empty object and `()` the empty array. The + marker is part of the notation, so the empty key round-trips as `(o: ("" 2))`. + The single-line spelling of every shared fixture is pinned in + `fixtures/readable-format/cases.json`, so all four languages write the same + bytes. +- The `examples/append_only_log.py` example, which writes, counts, greps and + reads back a log of one record per line. + +### Fixed + +- `decode` no longer routes a readable single-line document such as `(None 1)` + to the compact (base64) reader. Only a compact document keeps that path; the + document `(None)` stays the compact `None` so older documents keep decoding. diff --git a/rust/changelog.d/20260827_090000_issue_43_single_line_format.md b/rust/changelog.d/20260827_090000_issue_43_single_line_format.md new file mode 100644 index 0000000..73faa1b --- /dev/null +++ b/rust/changelog.d/20260827_090000_issue_43_single_line_format.md @@ -0,0 +1,25 @@ +--- +bump: minor +--- + +### Added + +- `encode_line` and `decode_line`: the readable format written on one line, so + an append-only log holds one record per line. Appending is one write, + compaction cuts at a newline, and `grep`, `tail -f` and `wc -l` treat a line + as one event. The output is valid Links Notation, keeps numbers, booleans and + `null` bare so types survive the round trip, and `decode(encode_line(v))` + equals `decode(encode(v))`. See + [issue #43](https://github.com/link-foundation/lino-objects-codec/issues/43). +- `readable::OBJECT_MARKER`, the `o` link id that tells an object from an array + on one line: `(o: (bytes 2827) (complete true))` is a record, `("a" 1)` is a + two-element array, `(o:)` is the empty object and `()` the empty array. The + marker is part of the notation, so the empty key round-trips as `(o: ("" 2))`. +- The `append_only_log` example, which writes, counts, greps and reads back a + log of one record per line. + +### Fixed + +- `decode` no longer routes a readable single-line document such as `(null 1)` + to the compact (base64) reader. Only a compact document keeps that path; the + document `(null)` stays the compact null so older documents keep decoding. From 83dd08767b898e38b111656d218d998091c07304 Mon Sep 17 00:00:00 2001 From: konard Date: Thu, 27 Aug 2026 12:35:07 +0000 Subject: [PATCH 12/12] fix(js): trim a record's framing newlines in linear time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeQL flagged `js/polynomial-redos` on `decodeLine`: the trailing trim `/[\n\r]+$/` restarts inside a run of newlines, so a record followed by many of them costs quadratic time — 20 s for 100k newlines, measured. Trim by scanning from both ends instead, which is what the Python, Rust and C# siblings already do with their built-in character trims. Each language gains the regression test: the padded record is still refused for holding more than one line, and returns well inside two seconds. --- .../SingleLineFormatTests.cs | 15 ++++++++++++ js/src/readable.js | 23 ++++++++++++++++++- js/tests/test_single_line_format.test.js | 12 ++++++++++ python/tests/test_single_line_format.py | 13 +++++++++++ rust/tests/single_line_format.rs | 12 ++++++++++ 5 files changed, 74 insertions(+), 1 deletion(-) diff --git a/csharp/tests/Lino.Objects.Codec.Tests/SingleLineFormatTests.cs b/csharp/tests/Lino.Objects.Codec.Tests/SingleLineFormatTests.cs index d399344..5bc603a 100644 --- a/csharp/tests/Lino.Objects.Codec.Tests/SingleLineFormatTests.cs +++ b/csharp/tests/Lino.Objects.Codec.Tests/SingleLineFormatTests.cs @@ -194,6 +194,21 @@ public void ALineStartingWithNullIsStillReadAsALine() Assert.Null(Codec.Decode("(null)")); } + /// + /// The JavaScript sibling trimmed the framing newlines with a regular + /// expression that backtracked once per newline (CodeQL js/polynomial-redos). + /// Every language strips them with a linear scan instead, and still refuses + /// input holding more than one line. + /// + [Fact] + public void ALongRunOfLineBreaksIsRejectedWithoutASlowdown() + { + var notation = Codec.EncodeLine(LogRecord()) + new string('\n', 200_000) + "x"; + var started = System.Diagnostics.Stopwatch.StartNew(); + Assert.Throws(() => Codec.DecodeLine(notation)); + Assert.True(started.Elapsed < TimeSpan.FromSeconds(2), $"took {started.Elapsed}"); + } + /// Structural comparison, since dictionaries and lists compare by reference. private static bool Equivalent(object? left, object? right) { diff --git a/js/src/readable.js b/js/src/readable.js index 2d2150b..c5d3649 100644 --- a/js/src/readable.js +++ b/js/src/readable.js @@ -146,6 +146,27 @@ export function encodeLine(value) { return out.join(''); } +/** + * Strip the line breaks framing a record, without a regular expression. + * + * A regular expression anchored at the end backtracks over a run of newlines, + * so a long run of them costs more than linear time. Scanning from both ends + * costs one pass over the framing characters. + * @param {string} text - The text to trim + * @returns {string} The text without leading or trailing newlines + */ +function trimLineBreaks(text) { + let start = 0; + let end = text.length; + while (start < end && (text[start] === '\n' || text[start] === '\r')) { + start += 1; + } + while (end > start && (text[end - 1] === '\n' || text[end - 1] === '\r')) { + end -= 1; + } + return text.slice(start, end); +} + /** * Decode the readable, single-line Links Notation form back into a value. * @@ -157,7 +178,7 @@ export function encodeLine(value) { * @throws {SyntaxError} If the input holds more than one line */ export function decodeLine(text) { - const line = text.replace(/^[\n\r]+/, '').replace(/[\n\r]+$/, ''); + const line = trimLineBreaks(text); if (line.includes('\n') || line.includes('\r')) { throw new SyntaxError('a single-line document cannot contain a line break'); } diff --git a/js/tests/test_single_line_format.test.js b/js/tests/test_single_line_format.test.js index c1739be..8490c67 100644 --- a/js/tests/test_single_line_format.test.js +++ b/js/tests/test_single_line_format.test.js @@ -114,3 +114,15 @@ test('a line starting with null is still read as a line', () => { // read that way, so documents written before this format keep decoding. assert.equal(decode({ notation: '(null)' }), null); }); + +test('a long run of line breaks is rejected without a slowdown', () => { + // CodeQL alert js/polynomial-redos: trimming the framing newlines with + // `/[\n\r]+$/` backtracked once per newline, so a record followed by a long + // run of them cost quadratic time. The scan that replaced it is linear, and + // the input is still refused for holding more than one line. + const notation = `${LOG_RECORD_LINE}${'\n'.repeat(200000)}x`; + const started = process.hrtime.bigint(); + assert.throws(() => decodeLine({ notation }), SyntaxError); + const elapsedMs = Number(process.hrtime.bigint() - started) / 1e6; + assert.ok(elapsedMs < 2000, `took ${elapsedMs}ms`); +}); diff --git a/python/tests/test_single_line_format.py b/python/tests/test_single_line_format.py index a6887ff..02449fb 100644 --- a/python/tests/test_single_line_format.py +++ b/python/tests/test_single_line_format.py @@ -7,6 +7,7 @@ hides it in base64, so neither serves that reader. """ +import time from typing import Any import pytest @@ -122,3 +123,15 @@ def test_a_line_starting_with_none_is_still_read_as_a_line() -> None: # language writes, and stays read that way, so documents written before this # format keep decoding. assert decode("(None)") is None + + +def test_a_long_run_of_line_breaks_is_rejected_without_a_slowdown() -> None: + # The JavaScript sibling trimmed the framing newlines with a regular + # expression that backtracked once per newline (CodeQL js/polynomial-redos). + # Every language strips them with a linear scan instead, and still refuses + # input holding more than one line. + notation = LOG_RECORD_LINE + "\n" * 200_000 + "x" + started = time.perf_counter() + with pytest.raises(ReadableFormatError): + decode_line(notation) + assert time.perf_counter() - started < 2.0 diff --git a/rust/tests/single_line_format.rs b/rust/tests/single_line_format.rs index 9ab2e83..489bdb3 100644 --- a/rust/tests/single_line_format.rs +++ b/rust/tests/single_line_format.rs @@ -176,3 +176,15 @@ fn a_line_starting_with_null_is_still_read_as_a_line() { // read that way, so documents written before this format keep decoding. assert_eq!(decode("(null)"), Ok(LinoValue::Null)); } + +/// The JavaScript sibling trimmed the framing newlines with a regular +/// expression that backtracked once per newline (CodeQL js/polynomial-redos). +/// Every language strips them with a linear scan instead, and still refuses +/// input holding more than one line. +#[test] +fn a_long_run_of_line_breaks_is_rejected_without_a_slowdown() { + let notation = format!("{}{}x", encode_line(&log_record()), "\n".repeat(200_000)); + let started = std::time::Instant::now(); + assert!(decode_line(¬ation).is_err()); + assert!(started.elapsed() < std::time::Duration::from_secs(2)); +}