From 9d2a26c2ef267a4379c7cddad42f01e9535f1972 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:30:57 +0100 Subject: [PATCH 1/2] Add RFC 7464 JSON text sequences to JSONStreaming JSONTextSequence reads application/json-seq texts framed by the record separator with the NDJSON pull model and budgets. Unterminated top-level scalars count as possibly truncated (RFC 7464 section 2.4). Malformed texts fail by default or are skipped and counted on request; resource failures are never skipped. JSONTextSequenceWriter writes RS, strict JSON and LF per value, as one sequence or in awaited bounded chunks. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 3 +- CHANGELOG.md | 7 + .../Qualification/TestDiscovery.json | 6 +- README.md | 4 +- Sources/JSONCLIEngine/CLIBuildIdentity.swift | 2 +- .../Documentation.docc/JSONStreaming.md | 12 +- .../Documentation.docc/TextSequences.md | 54 ++++++++ .../TextSequence/JSONTextSequence.swift | 120 ++++++++++++++++++ .../JSONTextSequenceOptions.swift | 26 ++++ .../TextSequence/JSONTextSequenceWriter.swift | 107 ++++++++++++++++ .../TextSequenceTests.swift | 102 +++++++++++++++ 11 files changed, 436 insertions(+), 7 deletions(-) create mode 100644 Sources/JSONStreaming/Documentation.docc/TextSequences.md create mode 100644 Sources/JSONStreaming/TextSequence/JSONTextSequence.swift create mode 100644 Sources/JSONStreaming/TextSequence/JSONTextSequenceOptions.swift create mode 100644 Sources/JSONStreaming/TextSequence/JSONTextSequenceWriter.swift create mode 100644 Tests/JSONStreamingTests/TextSequenceTests.swift diff --git a/AGENTS.md b/AGENTS.md index 91251a9..3eb2c7a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,7 +57,8 @@ store/follow API on 2026-11-10; later API changes stay additive and human-gated. one invocation-owned ledger per operation. Positions remain unforgeable and source-instance-local. No persistence or mutable store is exposed. See Documentation/Architecture/StructuralIntelligence/BoundedWorkflows1.8.md. -Wider retrieval/persistence and JSON text-sequence framing remain separate work. +Wider retrieval/persistence remains separate work. RFC 7464 JSON text sequences +are 1.10 development in JSONStreaming (`JSONTextSequence`, `JSONTextSequenceWriter`). JSONTransform uses one private compiler/VM and the CLI's accounted adapter. Preserve its accepted 42 authored declarations and exact semantics. The original diff --git a/CHANGELOG.md b/CHANGELOG.md index dd015f9..f49d90e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,13 @@ complete the page. Its CSP authorizes the two stylesheets by hash and sets `script-src 'none'`. The comparison is a package walker in JSON shared with `JSONPatch.difference`, so JSONHTML still depends on JSON only. +- Add RFC 7464 JSON text sequences (`application/json-seq`) to JSONStreaming. + `JSONTextSequence` reads texts framed by the record separator byte with the + same pull model and budgets as NDJSON. Unterminated top-level scalars are + treated as possibly truncated (RFC 7464 section 2.4). Malformed texts fail by + default; `JSONTextSequenceOptions.MalformedTexts.skip` skips them and counts + them on the iterator, but never skips resource failures. `JSONTextSequenceWriter` + writes RS, strict JSON and LF per value, as one sequence or in awaited chunks. ## Unreleased — 1.9.1 hygiene diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index cc5bedf..36b3a4a 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -4,7 +4,7 @@ "JSONTests": 198, "JSONSchemaTests": 79, "JSONRegexRuntimeTests": 16, - "JSONStreamingTests": 36, + "JSONStreamingTests": 42, "JSONPatchTests": 22, "JSONIntegrationTests": 45, "JSONHTMLTests": 103, @@ -76,7 +76,7 @@ "JSONTests": 198, "JSONSchemaTests": 79, "JSONRegexRuntimeTests": 16, - "JSONStreamingTests": 36, + "JSONStreamingTests": 42, "JSONPatchTests": 22, "JSONIntegrationTests": 45, "JSONHTMLTests": 87, @@ -92,7 +92,7 @@ "JSONTests": 198, "JSONSchemaTests": 79, "JSONRegexRuntimeTests": 16, - "JSONStreamingTests": 36, + "JSONStreamingTests": 42, "JSONPatchTests": 22, "JSONIntegrationTests": 45, "JSONHTMLTests": 87, diff --git a/README.md b/README.md index 3aca73f..a56826a 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Choose the products your application needs: | **JSON** | Exact values, strict parsing/writing, native Codable, Pointer, presence, checked construction, source documents and diagnostics | | **JSONSchema** | Draft 2020-12 compilation/evaluation, offline registries, bounded regex, annotations and raw declaration inspection | | **JSONSchemaExamples** | Opt-in deterministic exact candidates, conservative typed construction and native schema certification, introduced in 1.6.0 | -| **JSONStreaming** | Pull syntax events and values, strict NDJSON, root-array elements and awaited output chunks | +| **JSONStreaming** | Pull syntax events and values, strict NDJSON, RFC 7464 text sequences, root-array elements and awaited output chunks | | **JSONHTML** | Exact HTML and immutable exports; structured forms/cards and bounded previews in 1.6.0 | | **swift-json** | Bounded command-line JSON workflows (available since 1.5.0) | | **JSONBridges** | Opt-in native integer, Double and Foundation Decimal conversions (available in the 1.4.0 tag and 1.5.0 release) | @@ -471,6 +471,8 @@ producer queue. Root-array elements and syntax events remain provisional until completion. A document completes only after its closing syntax, trailing whitespace and EOF. `JSONValueSequence` waits for its framing boundary before yielding a value; `NDJSONValueSequence` rejects blank/malformed physical records and never skips them. +`JSONTextSequence` reads RFC 7464 `application/json-seq` texts; it treats unterminated +top-level scalars as possibly truncated and skips malformed texts only when asked. Since 1.8.0, `JSONLocatedTopLevelArraySequence` drives the same root-array reader and also yields each element's `JSONSourceSpan`: zero-based half-open UTF-8 byte offsets in the complete stream, excluding whitespace and separators. It retains no original diff --git a/Sources/JSONCLIEngine/CLIBuildIdentity.swift b/Sources/JSONCLIEngine/CLIBuildIdentity.swift index 5e2905f..1810a62 100644 --- a/Sources/JSONCLIEngine/CLIBuildIdentity.swift +++ b/Sources/JSONCLIEngine/CLIBuildIdentity.swift @@ -2,6 +2,6 @@ // Generated by Scripts/generate_cli_identity.py; excludes this file. // SHA-256 of the ordered Package.swift + Sources content manifest (DocC catalogs excluded), not a Git commit. package enum CLIBuildIdentity { - package static let revision = "sha256:dafab040289a71fed6924ffe8aa7a7273e6315c5701d1fd35fd5d1b812a52ccd" + package static let revision = "sha256:2751b97e2a61920fc9a31928764ed8af9c54d478fcab49050896aed00cfbaaec" package static let kind = "source-sha256" } diff --git a/Sources/JSONStreaming/Documentation.docc/JSONStreaming.md b/Sources/JSONStreaming/Documentation.docc/JSONStreaming.md index 0c241ff..1594c08 100644 --- a/Sources/JSONStreaming/Documentation.docc/JSONStreaming.md +++ b/Sources/JSONStreaming/Documentation.docc/JSONStreaming.md @@ -1,6 +1,6 @@ # ``JSONStreaming`` -Pull-driven events and values, strict NDJSON, root-array elements and awaited sinks. +Pull-driven events and values, strict NDJSON, RFC 7464 text sequences, root-array elements and awaited sinks. Sources are caller-owned AsyncSequences of owned ``JSONByteChunk`` values. Each iterator has exclusive mutable state, performs one pull at a time and @@ -19,6 +19,12 @@ blank/malformed records and accepts a valid final record without a final LF. There is no skipped-record recovery. Error locations include absolute byte and physical record positions. +``JSONTextSequence`` reads RFC 7464 JSON text sequences (`application/json-seq`): +each text starts with a record separator byte. Unterminated top-level scalars +are treated as possibly truncated. Malformed texts fail by default and can be +skipped explicitly. ``JSONTextSequenceWriter`` writes the same framing. See +. + ``JSONTopLevelArraySequence`` emits indexed elements followed by one explicit document-completed summary. Earlier elements remain provisional with respect to the enclosing document. A malformed/truncated tail produces no completion. @@ -43,6 +49,7 @@ publication must stage bytes in its own sink. ### Guides - +- ### Pull sequences - ``JSONByteChunk`` @@ -55,6 +62,8 @@ publication must stage bytes in its own sink. ### Record framing and completion - ``NDJSONValueSequence`` - ``NDJSONOptions`` +- ``JSONTextSequence`` +- ``JSONTextSequenceOptions`` - ``JSONTopLevelArraySequence`` - ``JSONTopLevelArrayEvent`` - ``JSONLocatedTopLevelArraySequence`` @@ -62,3 +71,4 @@ publication must stage bytes in its own sink. ### Awaited output - ``JSONAsyncWriter`` +- ``JSONTextSequenceWriter`` diff --git a/Sources/JSONStreaming/Documentation.docc/TextSequences.md b/Sources/JSONStreaming/Documentation.docc/TextSequences.md new file mode 100644 index 0000000..01d875e --- /dev/null +++ b/Sources/JSONStreaming/Documentation.docc/TextSequences.md @@ -0,0 +1,54 @@ +# JSON text sequences + +Read and write RFC 7464 `application/json-seq` streams. + +## Overview + +A JSON text sequence frames each JSON text with a leading record separator byte +(RS, `0x1E`) and, when written, a trailing LF. Because RS can never appear +unescaped inside JSON, a reader can find the next text even after a malformed +one. That makes the format useful for logs and long-running streams where one +damaged record should not hide the rest. + +``JSONTextSequence`` reads texts from caller-owned chunks with the same pull +model, cumulative iterator budget and per-text value quotas as +``NDJSONValueSequence``: + +```swift +for try await value in JSONTextSequence(chunks) { + handle(value) +} +``` + +Each text runs from an RS to the next RS or the end of input and is parsed by +the strict grammar, so surrounding whitespace is allowed and a text may span +several lines. Consecutive RS bytes are not empty texts. + +### Truncated and malformed texts + +A top-level number, `true`, `false` or `null` that does not end in whitespace +may have been cut short ("123" could be the start of "1234"), so RFC 7464 +section 2.4 treats it as possibly truncated. The reader reports it as an +`unexpectedEOF` syntax failure. Bytes before the first RS are malformed as well. + +By default a malformed text ends the sequence with its failure, located by +absolute byte offset, text number (`record`) and line and column within the +text. With ``JSONTextSequenceOptions/MalformedTexts/skip`` the reader skips +malformed texts and continues with the next RS, as RFC 7464 recommends; +``JSONTextSequence/Iterator/skippedTexts`` counts them. Resource failures, +cancellation and source errors are never skipped. + +### Writing + +``JSONTextSequenceWriter`` writes each value as RS, the strict writer's bytes +and LF. ``JSONTextSequenceWriter/encode(_:)`` produces a complete sequence under +one ledger. ``JSONTextSequenceWriter/write(_:to:)`` writes one text in awaited, +bounded chunks, like ``JSONAsyncWriter``; call it once per value. Bytes a sink +has accepted cannot be recalled, but a reader recognizes a partially written +text as malformed. + +## Topics + +- ``JSONTextSequence`` +- ``JSONTextSequenceOptions`` +- ``JSONTextSequenceWriter`` diff --git a/Sources/JSONStreaming/TextSequence/JSONTextSequence.swift b/Sources/JSONStreaming/TextSequence/JSONTextSequence.swift new file mode 100644 index 0000000..c2e9d33 --- /dev/null +++ b/Sources/JSONStreaming/TextSequence/JSONTextSequence.swift @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 +// Original RFC 7464 (application/json-seq) reader over the shared stream input. +import JSON +/// A pull reader for RFC 7464 JSON text sequences (`application/json-seq`). +/// +/// Each text starts with the record separator byte (RS, `0x1E`) and runs to the +/// next RS or the end of input. Consecutive RS bytes are not empty texts. Each text +/// is parsed by the strict UTF-8 grammar, so surrounding whitespace, including the +/// LF that writers append, is allowed. +/// +/// A text that is a top-level number, `true`, `false` or `null` must end in +/// whitespace; otherwise it may have been truncated and is malformed (RFC 7464 +/// section 2.4). Bytes before the first RS are a malformed text too. Malformed +/// texts end the sequence by default; ``JSONTextSequenceOptions/MalformedTexts/skip`` +/// continues with the next text instead. +/// +/// Failures report the absolute byte offset, the one-based text number as `record`, +/// and the line and column within the text. +public struct JSONTextSequence: AsyncSequence where Source.Element == JSONByteChunk { + /// Each complete JSON text, in order. + public typealias Element = JSON + let source: Source + let options: JSONTextSequenceOptions + /// Wraps a caller-owned source of `application/json-seq` bytes. + /// + /// - Parameters: + /// - source: The caller-owned asynchronous source of owned UTF-8 chunks. + /// - options: Limits and the malformed-text policy. + public init(_ source: Source, options: JSONTextSequenceOptions = .init()) { self.source = source; self.options = options } + /// Creates an independent iterator with exclusive mutable state and no prefetch task. + public func makeAsyncIterator() -> Iterator { .init(source: source.makeAsyncIterator(), options: options) } + /// Exclusive pull iterator; never call next concurrently. + /// + /// Each iterator owns its budget and parsing state. Conditional Sendable of the + /// sequence does not make this mutable iterator safe for concurrent access. + public struct Iterator: AsyncIteratorProtocol { + private var source: Source.AsyncIterator? + private let options: JSONTextSequenceOptions + private var driver: JSONTextSequenceDriver? + fileprivate init(source: Source.AsyncIterator, options: JSONTextSequenceOptions) { self.source = source; self.options = options } + /// The number of malformed texts skipped so far; always zero with the `fail` policy. + public var skippedTexts: Int { driver?.skipped ?? 0 } + /// Pulls at most one value through exclusively owned iterator state. + /// + /// An error or cancellation makes an initialized iterator terminal and prevents later pulls. + /// - Returns: The next JSON text's value, or nil at the end of input. + /// - Throws: Source, syntax or resource errors, or cancellation. + public mutating func next() async throws -> JSON? { + if driver == nil { driver = try JSONTextSequenceDriver(upstream: source!, options: options); source = nil } + return try await driver!.next() + } + } +} +extension JSONTextSequence: Sendable where Source: Sendable {} + +private final class JSONTextSequenceDriver where Upstream.Element == JSONByteChunk { + static var separator: UInt8 { 0x1E } + let input: JSONStreamInput + let skipMalformed: Bool + var terminal = false, resetOnNext = false, afterSeparator = false + var offset: UInt64 = 0, texts: UInt64 = 0 + var skipped = 0 + init(upstream: Upstream, options: JSONTextSequenceOptions) throws { + input = try JSONStreamInput(upstream: upstream, limits: options.limits); skipMalformed = options.malformedTexts == .skip + } + func next() async throws -> JSON? { + guard !terminal else { return nil } + do { + try input.meter.checkpoint(); try input.meter.require(.pendingValues, 1) + while true { + if resetOnNext { try input.resetRecord(); resetOnNext = false } + let introduced = afterSeparator, start = offset + var text: [UInt8] = [], ended = false + while true { + guard try await input.ensureBytes() else { ended = true; break } + let byte = input.bytes[input.index]; input.index += 1; offset += 1 + try input.meter.work() + if byte == Self.separator { break } + try input.meter.require(.inputBytes, UInt64(text.count) + 1); try input.meter.retain(1); text.append(byte) + } + afterSeparator = !ended + // Empty segments: input that starts with RS, consecutive RS bytes, or a final RS. + if text.isEmpty { + if ended { terminal = true; return nil } + continue + } + texts += 1; resetOnNext = true + do { + let value = try parse(text, introduced: introduced) + try input.meter.charge(.totalRecords) + if ended { terminal = true } + return value + } catch let failure as JSONFailure { + let relative = failure.location?.byteOffset ?? 0 + let located = JSONFailure(failure.category, code: failure.code, + location: .init(byteOffset: start + relative, line: failure.location?.line, + column: failure.location?.column, record: texts), + resource: failure.resource) + guard skipMalformed, failure.category == .syntax else { throw located } + try input.meter.charge(.totalRecords); skipped += 1 + if ended { terminal = true; return nil } + } + } + } catch { terminal = true; input.failed = true; throw error } + } + /// Parses one text; a missing leading RS or a possibly truncated scalar is a syntax failure. + private func parse(_ text: [UInt8], introduced: Bool) throws -> JSON { + guard introduced else { throw JSONFailure(.syntax, code: .invalidSyntax, location: .init(byteOffset: 0, line: 1, column: 1)) } + let value = try JSONParser().parse(text, meter: &input.meter) + switch value { + case .object, .array, .string: return value + default: + // RFC 7464 section 2.4: a top-level scalar without trailing whitespace may be truncated. + guard let last = text.last, last == 0x20 || last == 0x09 || last == 0x0A || last == 0x0D else { + throw JSONFailure(.syntax, code: .unexpectedEOF, location: .init(byteOffset: UInt64(text.count))) + } + return value + } + } +} diff --git a/Sources/JSONStreaming/TextSequence/JSONTextSequenceOptions.swift b/Sources/JSONStreaming/TextSequence/JSONTextSequenceOptions.swift new file mode 100644 index 0000000..efc0593 --- /dev/null +++ b/Sources/JSONStreaming/TextSequence/JSONTextSequenceOptions.swift @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +/// Finite limits and the malformed-text policy for RFC 7464 JSON text sequences. +public struct JSONTextSequenceOptions: Sendable { + /// What a reader does with a text that is not one complete JSON value. + public enum MalformedTexts: Sendable, Equatable { + /// Throw the syntax failure and end the sequence (the default). + case fail + /// Skip the text and continue with the next record separator, as RFC 7464 + /// recommends. Resource failures, cancellation and source errors still end + /// the sequence. The iterator counts skipped texts. + case skip + } + /// Finite capacities and work quotas applied to library-owned operations. + public var limits: JSONLimits + /// The policy for texts that fail to parse or are possibly truncated. + public var malformedTexts: MalformedTexts + /// Creates the configuration; finite limits are checked at operation entry. + /// + /// - Parameters: + /// - limits: Finite inclusive capacities and work quotas, per text and for the iterator lifetime. + /// - malformedTexts: Whether malformed texts end the sequence or are skipped. + public init(limits: JSONLimits = .balanced, malformedTexts: MalformedTexts = .fail) { + self.limits = limits; self.malformedTexts = malformedTexts + } +} diff --git a/Sources/JSONStreaming/TextSequence/JSONTextSequenceWriter.swift b/Sources/JSONStreaming/TextSequence/JSONTextSequenceWriter.swift new file mode 100644 index 0000000..5342830 --- /dev/null +++ b/Sources/JSONStreaming/TextSequence/JSONTextSequenceWriter.swift @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: Apache-2.0 +// Original RFC 7464 (application/json-seq) writer over the strict writing cursor. +import Foundation +import JSON +/// Writes RFC 7464 JSON text sequences (`application/json-seq`). +/// +/// Every value becomes one text: the record separator byte (RS, `0x1E`), the strict +/// JSON bytes of ``JSONWritingOptions``, then one LF. The trailing LF lets readers +/// detect truncated top-level scalars. Indented output is allowed, because the +/// LF bytes inside a text are JSON whitespace. +public struct JSONTextSequenceWriter: Sendable { + /// The strict writing configuration used for each text. + public let options: JSONWritingOptions + /// Maximum bytes per awaited handoff; must be positive and fit the chunkBytes limit. + public let chunkSize: Int + /// Configures the writer; positive chunk size and finite capacity are checked on write. + /// + /// - Parameters: + /// - options: The strict writing configuration and finite limits. + /// - chunkSize: Maximum bytes per awaited handoff; validated at write entry. + public init(options: JSONWritingOptions = .init(), chunkSize: Int = 16_384) { self.options = options; self.chunkSize = chunkSize } + + /// Encodes the values as one complete sequence. + /// + /// - Returns: The sequence bytes; an empty array gives empty data. + /// - Throws: Resource or configuration failure, or cancellation. + /// + /// - Parameters: + /// - values: The exact roots, written in order, one text each. + public func encode(_ values: [JSON]) throws -> Data { try encodeWithReport(values).result } + /// Encodes the values with one cumulative work ledger for the whole sequence. + /// + /// - Returns: The sequence bytes and the ledger's charged counts. + /// - Throws: Resource or configuration failure, or cancellation. + /// + /// - Parameters: + /// - values: The exact roots, written in order, one text each. + public func encodeWithReport(_ values: [JSON]) throws -> JSONOperationReport { + var meter = try configuredMeter() + var bytes: [UInt8] = [] + for value in values { + try meter.checkpoint(); try meter.resetValueBounds() + try text(value, meter: &meter) { byte, meter in try meter.retain(1); bytes.append(byte) } + } + try meter.checkpoint() + return .init(result: Data(bytes), statistics: meter.statistics()) + } + + /// Writes one value as one text by awaiting full acceptance of each bounded owned chunk. + /// + /// Call once per value to produce a sequence. No later handoff occurs after a sink + /// error or cancellation; accepted bytes cannot be recalled, and readers skip or + /// reject the partial text. + /// - Throws: Sink errors, resource/configuration failure or cancellation. + /// + /// - Parameters: + /// - value: The exact root to admit and write as one text. + /// - sink: The awaited callback; successful return accepts the entire owned chunk. + public func write(_ value: JSON, to sink: @Sendable (JSONByteChunk) async throws -> Void) async throws { + _ = try await writeWithReport(value, to: sink) + } + /// Writes one text in awaited chunks and returns successful output accounting. + /// + /// - Returns: Counts only after every handoff and final cancellation checkpoint succeeds. + /// - Throws: Sink errors, resource/configuration failure or cancellation; accepted bytes cannot be recalled. + /// + /// - Parameters: + /// - value: The exact root to admit and write as one text. + /// - sink: The awaited callback; successful return accepts the entire owned chunk. + public func writeWithReport(_ value: JSON, to sink: @Sendable (JSONByteChunk) async throws -> Void) async throws -> JSONOperationStatistics { + var meter = try configuredMeter() + guard chunkSize > 0, UInt64(chunkSize) <= options.limits[.chunkBytes] else { throw JSONFailure(.configuration, code: .invalidLimits) } + try JSONAdmission.value(value, meter: &meter) + var cursor = JSONWritingCursor(value: value, options: options), bytes: [UInt8] = [] + var phase = 0 // 0: RS, 1: JSON bytes, 2: LF, 3: done + while phase < 3 { + let byte: UInt8 + switch phase { + case 0: byte = 0x1E; phase = 1 + case 1: + guard let next = try cursor.next(meter: &meter) else { phase = 2; continue } + byte = next + default: byte = 0x0A; phase = 3 + } + try meter.charge(.outputBytes); try meter.retain(1); bytes.append(byte) + if bytes.count == chunkSize || (phase == 3 && !bytes.isEmpty) { + try meter.checkpoint(); try await sink(JSONByteChunk(bytes)); try meter.checkpoint() + try meter.release(UInt64(bytes.count)); bytes = [] + } + } + return meter.statistics() + } + + private func configuredMeter() throws -> JSONWorkMeter { + let meter = try JSONWorkMeter(limits: options.limits, domain: .writing) + guard options.indentation.map({ (0...8).contains($0) }) ?? true else { throw JSONFailure(.configuration, code: .invalidLimits) } + return meter + } + /// Emits RS, the strict JSON bytes and LF, charging every output byte. + private func text(_ value: JSON, meter: inout JSONWorkMeter, emit: (UInt8, inout JSONWorkMeter) throws -> Void) throws { + try JSONAdmission.value(value, meter: &meter) + try meter.charge(.outputBytes); try emit(0x1E, &meter) + var cursor = JSONWritingCursor(value: value, options: options) + while let byte = try cursor.next(meter: &meter) { try meter.charge(.outputBytes); try emit(byte, &meter) } + try meter.charge(.outputBytes); try emit(0x0A, &meter) + } +} diff --git a/Tests/JSONStreamingTests/TextSequenceTests.swift b/Tests/JSONStreamingTests/TextSequenceTests.swift new file mode 100644 index 0000000..bca688b --- /dev/null +++ b/Tests/JSONStreamingTests/TextSequenceTests.swift @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Testing +import JSON +import JSONStreaming + +private let rs = "\u{1E}" +private func parse(_ text: String) throws -> JSON { try JSONParser().parse(text) } +/// Reads every value and the skip count; `split` nil delivers one byte per chunk. +private func read(_ text: String, split: Int? = nil, skip: Bool = false, limits: JSONLimits = .balanced) async throws -> (values: [JSON], skipped: Int) { + let sequence = JSONTextSequence(Chunks(text, split: split), options: .init(limits: limits, malformedTexts: skip ? .skip : .fail)) + var iterator = sequence.makeAsyncIterator(), values: [JSON] = [] + while let value = try await iterator.next() { values.append(value) } + return (values, iterator.skippedTexts) +} +private func failure(_ text: String, skip: Bool = false, limits: JSONLimits = .balanced) async -> JSONFailure? { + do { _ = try await read(text, skip: skip, limits: limits); return nil } catch { return error as? JSONFailure } +} + +@Test func textSequenceReadsRecordsAcrossEveryChunkSplit() async throws { + let text = rs + "{\"a\":[1,2]}\n" + rs + "\"x\\u001e\"\n" + rs + "1.50\n" + rs + " [\n true\n]\r\n" + rs + "null\n" + let expected = [try parse(#"{"a":[1,2]}"#), .string("x\u{1E}"), try parse("1.50"), .array([.boolean(true)]), .null] + #expect(try await read(text).values == expected) + for split in 0...text.utf8.count { #expect(try await read(text, split: split).values == expected) } + // Consecutive, leading and final RS bytes are not empty texts; empty input is an empty sequence. + #expect(try await read(rs + rs + "1\n" + rs + rs + rs + "2\n" + rs).values == [1, 2]) + #expect(try await read("").values.isEmpty) + #expect(try await read(rs + rs).values.isEmpty) + // Containers and strings need no trailing LF. + #expect(try await read(rs + "[1]" + rs + "\"s\"").values == [.array([1]), .string("s")]) +} + +@Test func textSequenceTreatsUnterminatedScalarsAsTruncated() async throws { + // RFC 7464 section 2.4: "123" before the next RS may be a truncated "1234". + let truncated = await failure(rs + "123" + rs + "true\n") + #expect(truncated?.code == .unexpectedEOF && truncated?.category == .syntax) + #expect(truncated?.location?.byteOffset == 4 && truncated?.location?.record == 1) + for scalar in ["true", "false", "null", "-0.5", "42"] { + #expect(await failure(rs + scalar)?.code == .unexpectedEOF) + #expect(try await read(rs + scalar + " ").values.count == 1) + } + let skipped = try await read(rs + "123" + rs + "true\n" + rs + "7", skip: true) + #expect(skipped.values == [true] && skipped.skipped == 2) +} + +@Test func textSequenceRejectsOrSkipsMalformedTexts() async throws { + // Bytes before the first RS are a malformed text. + let leading = await failure("1\n" + rs + "2\n") + #expect(leading?.code == .invalidSyntax && leading?.location?.byteOffset == 0 && leading?.location?.record == 1) + // A syntax failure reports the absolute offset and the text number. + let broken = await failure(rs + "1\n" + rs + "{\"a\":}\n") + #expect(broken?.category == .syntax && broken?.location?.record == 2) + #expect((broken?.location?.byteOffset ?? 0) >= 4) + #expect(await failure(rs + "\n") != nil) + #expect(await failure(rs + "1 2\n")?.category == .syntax) + // The skip policy continues after each malformed text, as RFC 7464 recommends. + let recovered = try await read("junk" + rs + "{\"a\":" + rs + "[1]\n" + rs + "\u{FF}\n" + rs + "\"ok\"\n", skip: true) + #expect(recovered.values == [.array([1]), .string("ok")] && recovered.skipped == 3) + #expect(try await read(rs + "1\n").skipped == 0) +} + +@Test func textSequenceResourceFailuresAreNeverSkipped() async throws { + var limits = JSONLimits.balanced; limits[.inputBytes] = 8 + #expect(try await read(rs + "[1,2,3]\n" + rs + "[4]\n", limits: limits).values.count == 2) + let oversized = await failure(rs + "[1,2,3,4,5]\n" + rs + "1\n", skip: true, limits: limits) + #expect(oversized?.category == .resource && oversized?.resource?.metric == .inputBytes) + limits = .balanced; limits[.totalRecords] = 2 + #expect(await failure(rs + "1\n" + rs + "2\n" + rs + "3\n", limits: limits)?.resource?.metric == .totalRecords) +} + +@Test func textSequenceWriterProducesRFC7464Texts() async throws { + let values: [JSON] = [1, .string("x\u{1E}"), try parse(#"{"a":[],"n":1.50}"#)] + let data = try JSONTextSequenceWriter().encode(values) + // RS, the strict writer's exact bytes, then LF, per text. + #expect(String(decoding: data, as: UTF8.self) == (try values.map { rs + (try JSONWriter().writeString($0)) + "\n" }.joined())) + #expect(String(decoding: data, as: UTF8.self).hasPrefix(rs + "1\n" + rs + "\"x\\u001e\"\n" + rs + "{")) + #expect(try JSONTextSequenceWriter().encode([]).isEmpty) + // Indented texts round-trip because their LF bytes are JSON whitespace. + let pretty = try JSONTextSequenceWriter(options: .init(indentation: 2)).encode(values) + #expect(try await read(String(decoding: pretty, as: UTF8.self)).values == values) + let report = try JSONTextSequenceWriter().encodeWithReport(values) + #expect(report.statistics[.outputBytes] == UInt64(report.result.count)) + + // The awaited writer emits the same bytes in bounded chunks. + actor Collected { var chunks: [[UInt8]] = []; func add(_ chunk: JSONByteChunk) { chunks.append(chunk.bytes) } } + let collected = Collected() + for value in values { try await JSONTextSequenceWriter(chunkSize: 3).write(value) { await collected.add($0) } } + let chunks = await collected.chunks + #expect(chunks.allSatisfy { (1...3).contains($0.count) }) + #expect(Data(chunks.flatMap { $0 }) == data) + await #expect(throws: JSONFailure.self) { try await JSONTextSequenceWriter(chunkSize: 0).write(1) { _ in } } +} + +@Test func cancelledTextSequenceReadingThrowsCancellation() async { + let task = Task { + while !Task.isCancelled { await Task.yield() } + for try await _ in JSONTextSequence(Chunks(String(repeating: rs + "1\n", count: 200))) {} + } + task.cancel() + do { try await task.value; Issue.record("Cancelled reading must throw") } + catch { #expect(error is CancellationError) } +} From feee9c36ea035f751d69a2f5086f244672c5f0ee Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Sat, 14 Nov 2026 19:19:20 +0100 Subject: [PATCH 2/2] Use plain code for the cross-module JSONWritingOptions reference Co-Authored-By: Claude Opus 5.5 --- Sources/JSONCLIEngine/CLIBuildIdentity.swift | 2 +- Sources/JSONStreaming/TextSequence/JSONTextSequenceWriter.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/JSONCLIEngine/CLIBuildIdentity.swift b/Sources/JSONCLIEngine/CLIBuildIdentity.swift index 1810a62..86a1953 100644 --- a/Sources/JSONCLIEngine/CLIBuildIdentity.swift +++ b/Sources/JSONCLIEngine/CLIBuildIdentity.swift @@ -2,6 +2,6 @@ // Generated by Scripts/generate_cli_identity.py; excludes this file. // SHA-256 of the ordered Package.swift + Sources content manifest (DocC catalogs excluded), not a Git commit. package enum CLIBuildIdentity { - package static let revision = "sha256:2751b97e2a61920fc9a31928764ed8af9c54d478fcab49050896aed00cfbaaec" + package static let revision = "sha256:c0c4442af4a6e52b5e232fc41405acbf92616b366832ae0df0bfb843e18a3773" package static let kind = "source-sha256" } diff --git a/Sources/JSONStreaming/TextSequence/JSONTextSequenceWriter.swift b/Sources/JSONStreaming/TextSequence/JSONTextSequenceWriter.swift index 5342830..a172727 100644 --- a/Sources/JSONStreaming/TextSequence/JSONTextSequenceWriter.swift +++ b/Sources/JSONStreaming/TextSequence/JSONTextSequenceWriter.swift @@ -5,7 +5,7 @@ import JSON /// Writes RFC 7464 JSON text sequences (`application/json-seq`). /// /// Every value becomes one text: the record separator byte (RS, `0x1E`), the strict -/// JSON bytes of ``JSONWritingOptions``, then one LF. The trailing LF lets readers +/// JSON bytes of `JSONWritingOptions`, then one LF. The trailing LF lets readers /// detect truncated top-level scalars. Indented output is allowed, because the /// LF bytes inside a text are JSON whitespace. public struct JSONTextSequenceWriter: Sendable {