From de1fabb2c34ff7310dbf20165ecdbb474d4ef342 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:28:21 +0100 Subject: [PATCH] Add located root-array streaming with per-element source spans Add JSONLocatedTopLevelArraySequence and JSONLocatedTopLevelArrayEvent to JSONStreaming for 1.8.0-dev. Each root-array element also reports its JSONSourceSpan: zero-based half-open UTF-8 byte offsets in the complete stream, excluding whitespace and separators, independent of chunking. Both root-array facades share the one internal JSONArrayStreamDriver, so grammar, quotas, errors, statistics and completion are identical; the value-only next() delegates to nextLocated(). No bytes are retained. Add public-consumer tests for spans, UTF-8, chunk splits, empty arrays, facade parity on errors/limits and unchanged value-only behavior. Update DocC, architecture notes, CHANGELOG, test discovery counts, the parameterized-case registry and the regenerated CLI source identity. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 ++ Documentation/Architecture/API.md | 2 +- Documentation/Architecture/Streaming.md | 2 +- .../Qualification/TestDiscovery.json | 10 +- Scripts/testing_events.py | 2 +- Sources/JSONCLIEngine/CLIBuildIdentity.swift | 2 +- .../CompletionAndBackpressure.md | 29 ++++ .../Documentation.docc/JSONStreaming.md | 8 + .../JSONLocatedTopLevelArraySequence.swift | 55 +++++++ .../JSONTopLevelArraySequence.swift | 18 ++- .../LocatedTopLevelArrayTests.swift | 153 ++++++++++++++++++ 11 files changed, 280 insertions(+), 12 deletions(-) create mode 100644 Sources/JSONStreaming/JSONLocatedTopLevelArraySequence.swift create mode 100644 Tests/JSONStreamingTests/LocatedTopLevelArrayTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index bf9d7e6..87e6c59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## Unreleased — 1.8.0-dev + +- Add `JSONLocatedTopLevelArraySequence` and `JSONLocatedTopLevelArrayEvent` to + JSONStreaming. Each root-array element also reports its `JSONSourceSpan`: + zero-based half-open UTF-8 byte offsets in the complete stream, excluding + whitespace and separators, independent of chunk boundaries. Spans stay + provisional until `documentCompleted`; no original bytes are retained. +- Both root-array facades share one strict grammar and pull driver, so quotas, + errors, statistics and completion are identical. `JSONTopLevelArraySequence` + behavior and all existing products, dependencies and platforms are unchanged. + ## 1.7.1 — prepared source; publication separate - Fix packaged Draft 2020-12 resource discovery when SwiftPM's native test host diff --git a/Documentation/Architecture/API.md b/Documentation/Architecture/API.md index c8e167f..3a71eb0 100644 --- a/Documentation/Architecture/API.md +++ b/Documentation/Architecture/API.md @@ -30,7 +30,7 @@ and executed by public consumers. | JSONBridges | BinaryInteger.jsonNumber, FixedWidthInteger/Double/Decimal.init(jsonNumber:), Double.jsonNumberFromRoundTripText | Optional one-delegate numeric conversions; explicit fidelity, no new authority | | JSONSchema | JSONSchemaDocument, JSONSchemaRegistry, JSONSchemaCompiler, CompiledJSONSchema, JSONSchemaValidator | Offline immutable plan, invocation-owned evaluation and private native regex | | JSONSchemaExamples | JSONSchemaExampleGenerator, JSONSchemaExampleResult | Conservative deterministic witnesses certified by the existing native validator; separate finite preparation and cumulative generation | -| JSONStreaming | JSONEventSequence, JSONValueSequence, NDJSONValueSequence, JSONTopLevelArraySequence, JSONAsyncWriter | Exclusive pull state, shared codec, strict framing and awaited handoffs | +| JSONStreaming | JSONEventSequence, JSONValueSequence, NDJSONValueSequence, JSONTopLevelArraySequence, JSONLocatedTopLevelArraySequence, JSONAsyncWriter | Exclusive pull state, shared codec, strict framing and awaited handoffs | | JSONPatch | JSONPatch, JSONPatchOperation, JSONPatchResult, JSONMergePatch | Transaction staging, exact comparison and iterative editing | | swift-json | format, validate, pointer, patch, query, stream, schema, render | Bounded process/file adapters; the public Transform facade and CLI use the same private query compiler/VM | | JSONHTML | JSONHTMLRenderer, JSONHTMLRenderingOptions, JSONHTMLLimits, JSONHTMLRenderReport, JSONHTMLAnnotation, JSONExportDocument, JSONExportOptions, JSONExportReport | Structured forms/cards and bounded native previews, exact HTML, native Apple transfer, immutable prepared JSON/text/Markdown/static HTML/opt-in styled RTF and cumulative budgets | diff --git a/Documentation/Architecture/Streaming.md b/Documentation/Architecture/Streaming.md index b2656a9..6914e47 100644 --- a/Documentation/Architecture/Streaming.md +++ b/Documentation/Architecture/Streaming.md @@ -28,7 +28,7 @@ Large-stream tests use a generator that allocates one admitted chunk at a time a ## Root-array streaming -JSONTopLevelArraySequence returns an explicit array stream event: element(index, value) and documentCompleted(summary). It consumes one root array through the shared strict token machine and materializes only the current element. It has no selector language. +JSONTopLevelArraySequence returns an explicit array stream event: element(index, value) and documentCompleted(summary). It consumes one root array through the shared strict token machine and materializes only the current element. It has no selector language. JSONLocatedTopLevelArraySequence uses the same driver and yields element(index, value, sourceSpan) with zero-based half-open UTF-8 byte offsets in the complete stream; it retains no original bytes, and spans are provisional until documentCompleted. This serves large export files and datasets whose records are array elements rather than separate NDJSON lines. Each yielded value is a complete valid JSON subtree; the enclosing document is not certified until the closing bracket, trailing whitespace and EOF have all been checked. A malformed tail throws after any earlier yields. Early termination or cancellation never produces documentCompleted. diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index ee42d25..925216c 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -4,7 +4,7 @@ "JSONTests": 190, "JSONSchemaTests": 79, "JSONRegexRuntimeTests": 16, - "JSONStreamingTests": 30, + "JSONStreamingTests": 36, "JSONPatchTests": 16, "JSONIntegrationTests": 34, "JSONHTMLTests": 97, @@ -16,7 +16,7 @@ "JSONTransformTests": 23, "JSONDSLTests": 47 }, - "parameterized_invocations": 390, + "parameterized_invocations": 408, "composition_external_functions": 5, "qa_functions": { "JSONTests": 82, @@ -76,7 +76,7 @@ "JSONTests": 190, "JSONSchemaTests": 79, "JSONRegexRuntimeTests": 16, - "JSONStreamingTests": 30, + "JSONStreamingTests": 36, "JSONPatchTests": 16, "JSONIntegrationTests": 34, "JSONHTMLTests": 81, @@ -92,7 +92,7 @@ "JSONTests": 190, "JSONSchemaTests": 79, "JSONRegexRuntimeTests": 16, - "JSONStreamingTests": 30, + "JSONStreamingTests": 36, "JSONPatchTests": 16, "JSONIntegrationTests": 34, "JSONHTMLTests": 81, @@ -103,7 +103,7 @@ "JSONTransformTests": 23, "JSONDSLTests": 47 }, - "wasi_parameterized_invocations": 353, + "wasi_parameterized_invocations": 371, "portable_shared_state_functions": { "JSONTests": 4, "JSONSchemaTests": 3, diff --git a/Scripts/testing_events.py b/Scripts/testing_events.py index e552a99..34bb717 100644 --- a/Scripts/testing_events.py +++ b/Scripts/testing_events.py @@ -19,7 +19,7 @@ 'strictArrayIndexGrammar(text:)': 5, 'malformedUTF8(bytes:)': 4, 'invalidPointerEscapes(text:)': 4, 'invalidCancellationConfiguration(quantum:)': 3, 'strictMalformedInput(text:)': 10, 'negativeZeroHasOneIdentity(text:)': 3, - 'everyByteSplitUsesIdenticalGrammar(text:)': 4, 'malformedRootArrayNeverCompletes(text:)': 6, + 'everyByteSplitUsesIdenticalGrammar(text:)': 4, 'malformedRootArrayNeverCompletes(text:)': 6, 'locatedAndValueOnlyFacadesAgreeOnEveryOutcome(text:)': 18, 'streamEverySplitAgreesWithNativeParser(text:)': 3, 'successfulAlternativesCannotSwallowRegexExhaustion(keyword:)': 3, 'URIIdentityUsesSyntaxNormalizationWithoutUnicodeFolding(input:expected:)': 9, diff --git a/Sources/JSONCLIEngine/CLIBuildIdentity.swift b/Sources/JSONCLIEngine/CLIBuildIdentity.swift index 66f9355..b61044d 100644 --- a/Sources/JSONCLIEngine/CLIBuildIdentity.swift +++ b/Sources/JSONCLIEngine/CLIBuildIdentity.swift @@ -2,6 +2,6 @@ // Generated by Scripts/generate_cli_identity.py; excludes this file. // SHA-256 of the ordered Package.swift + Sources content manifest, not a Git commit. package enum CLIBuildIdentity { - package static let revision = "sha256:cc33adc50dbf28731ccac0919bfaf10716e459d3e69ffdd05189301d1d13b2f9" + package static let revision = "sha256:9d42eb6dbc4c0e23e7329ae1b3c278421d996c7446638e46cc5c1a07ab1240ac" package static let kind = "source-sha256" } diff --git a/Sources/JSONStreaming/Documentation.docc/CompletionAndBackpressure.md b/Sources/JSONStreaming/Documentation.docc/CompletionAndBackpressure.md index d526ae9..f3136e6 100644 --- a/Sources/JSONStreaming/Documentation.docc/CompletionAndBackpressure.md +++ b/Sources/JSONStreaming/Documentation.docc/CompletionAndBackpressure.md @@ -10,6 +10,7 @@ Choose framing, consume through its certified boundary and await output acceptan | JSONValueSequence | Complete values | Configured single-document or whitespace boundary | | NDJSONValueSequence | Complete physical records | LF/CRLF or valid final record at EOF | | JSONTopLevelArraySequence | Provisional indexed elements, then completion | Outer closing array, trailing whitespace and EOF | +| JSONLocatedTopLevelArraySequence | Provisional indexed elements with source spans, then completion | Outer closing array, trailing whitespace and EOF | Each reader uses owned ``JSONByteChunk`` values from a caller AsyncSequence. An iterator owns its mutable state and must be advanced serially. No detached @@ -32,6 +33,33 @@ results until completion. Consumers choosing incremental acceptance own their persistence rules. A single-document JSONValueSequence instead waits until EOF before yielding its whole materialized root. +## Locate root-array elements + +``JSONLocatedTopLevelArraySequence`` shares the root-array grammar, pull driver, +quotas, errors and completion boundary with ``JSONTopLevelArraySequence``. Each +``JSONLocatedTopLevelArrayEvent/element(index:value:sourceSpan:)`` also carries +the element's zero-based half-open UTF-8 byte span in the complete stream, +including any leading document whitespace before the array. Whitespace and +separators between elements are excluded; a container's span covers its whole +subtree. Offsets count encoded bytes, not characters or decoded scalars, and do +not depend on how the source splits chunks. + +The reader owns no copy of the original bytes. A consumer that needs a record's +raw slice retains an immutable source separately and applies the span to it. +Like the element itself, a span stays provisional until +``JSONLocatedTopLevelArrayEvent/documentCompleted(_:)``. + +```swift +for try await event in JSONLocatedTopLevelArraySequence(chunks) { + switch event { + case .element(let index, let value, let span): + stage(index, value, bytes[Int(span.start)..: AsyncSequence where Source.Element == JSONByteChunk { + /// The located event yielded by this sequence, as declared below. + public typealias Element = JSONLocatedTopLevelArrayEvent + let source: Source + let options: JSONStreamingOptions + /// Wraps a caller-owned source for one root array; framing is always single-document. + /// + /// - Parameters: + /// - source: The caller-owned asynchronous source of owned UTF-8 chunks. + /// - options: The value configuration used by this facade or operation. + public init(_ source: Source, options: JSONStreamingOptions = .init()) { self.source = source; self.options = options } + /// Creates an independent iterator with exclusive mutable state and no prefetch task. + public func makeAsyncIterator() -> Iterator { .init(source: source.makeAsyncIterator(), limits: options.limits) } + /// Exclusive pull iterator; never call next concurrently. + /// + /// Each iterator owns its budget, parsing state and span origin. Conditional + /// Sendable of the sequence does not make this mutable iterator safe for concurrent access. + public struct Iterator: AsyncIteratorProtocol { + private var source: Source.AsyncIterator? + private let limits: JSONLimits + private var driver: JSONArrayStreamDriver? + fileprivate init(source: Source.AsyncIterator, limits: JSONLimits) { self.source = source; self.limits = limits } + /// Pulls at most one located result through exclusively owned iterator state. + /// + /// Continue until the certified boundary or nil required by the sequence contract. + /// An error or cancellation makes an initialized iterator terminal and prevents later pulls. + /// - Returns: The next located event, or nil after termination. + /// - Throws: Source, syntax or resource errors, or cancellation. + public mutating func next() async throws -> Element? { + if driver == nil { driver = try JSONArrayStreamDriver(upstream: source!, limits: limits); source = nil } + return try await driver!.nextLocated() + } + } +} +extension JSONLocatedTopLevelArraySequence: Sendable where Source: Sendable {} diff --git a/Sources/JSONStreaming/JSONTopLevelArraySequence.swift b/Sources/JSONStreaming/JSONTopLevelArraySequence.swift index 97bf660..7a81bed 100644 --- a/Sources/JSONStreaming/JSONTopLevelArraySequence.swift +++ b/Sources/JSONStreaming/JSONTopLevelArraySequence.swift @@ -40,13 +40,23 @@ public struct JSONTopLevelArraySequence: AsyncSequence wh } } extension JSONTopLevelArraySequence: Sendable where Source: Sendable {} -private final class JSONArrayStreamDriver where Upstream.Element == JSONByteChunk { +final class JSONArrayStreamDriver where Upstream.Element == JSONByteChunk { let input: JSONStreamInput var machine = JSONTokenMachine(topLevelArrayStreaming: true) var assembler = JSONValueAssembler() var began = false, closed = false, terminal = false, resetOnNext = false + private var elementStart: UInt64? init(upstream: Upstream, limits: JSONLimits) throws { input = try JSONStreamInput(upstream: upstream, limits: limits) } func next() async throws -> JSONTopLevelArrayEvent? { + guard let event = try await nextLocated() else { return nil } + switch event { + case .element(let index, let value, _): return .element(index: index, value: value) + case .documentCompleted(let summary): return .documentCompleted(summary) + } + } + // Both facades share this one grammar and pull driver. Spans are absolute + // token offsets from the machine, never counts of delivered chunks or text. + func nextLocated() async throws -> JSONLocatedTopLevelArrayEvent? { guard !terminal else { return nil } do { try input.meter.checkpoint(); try input.meter.require(.pendingValues, 1) @@ -72,12 +82,14 @@ private final class JSONArrayStreamDriver where guard case .endArray = event.kind else { throw JSONFailure(.syntax, code: .invalidSyntax) } closed = true; continue } + if elementStart == nil { elementStart = event.start } if let value = try assembler.consume(event, meter: &input.meter) { let index = input.meter.used(.totalRecords) try input.meter.charge(.totalRecords); assembler = JSONValueAssembler(); resetOnNext = true - return .element(index: index, value: value) + let span = JSONSourceSpan(start: elementStart!, end: event.end); elementStart = nil + return .element(index: index, value: value, sourceSpan: span) } } - } catch { terminal = true; input.failed = true; assembler = JSONValueAssembler(); throw error } + } catch { terminal = true; input.failed = true; assembler = JSONValueAssembler(); elementStart = nil; throw error } } } diff --git a/Tests/JSONStreamingTests/LocatedTopLevelArrayTests.swift b/Tests/JSONStreamingTests/LocatedTopLevelArrayTests.swift new file mode 100644 index 0000000..e41d914 --- /dev/null +++ b/Tests/JSONStreamingTests/LocatedTopLevelArrayTests.swift @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import JSON +import JSONStreaming + +/// Everything one located iterator produced, in pull order. +private struct LocatedRun { + var elements: [(index: UInt64, value: JSON, span: JSONSourceSpan)] = [] + var completions: [JSONStreamSummary] = [] + var failure: JSONFailure? +} +/// Everything one value-only iterator produced, in pull order. +private struct PlainRun { + var elements: [(index: UInt64, value: JSON)] = [] + var completions: [JSONStreamSummary] = [] + var failure: JSONFailure? +} +private func locate(_ source: S, limits: JSONLimits = .balanced) async -> LocatedRun where S.Element == JSONByteChunk { + var run = LocatedRun() + var iterator = JSONLocatedTopLevelArraySequence(source, options: .init(limits: limits)).makeAsyncIterator() + do { + while let event = try await iterator.next() { + switch event { + case .element(let index, let value, let span): run.elements.append((index, value, span)) + case .documentCompleted(let summary): run.completions.append(summary) + } + } + } catch let failure as JSONFailure { run.failure = failure } catch { Issue.record("Unexpected error \(error)") } + // A finished or failed iterator stays terminal. + #expect((try? await iterator.next()) == nil) + return run +} +private func plain(_ source: S, limits: JSONLimits = .balanced) async -> PlainRun where S.Element == JSONByteChunk { + var run = PlainRun() + var iterator = JSONTopLevelArraySequence(source, options: .init(limits: limits)).makeAsyncIterator() + do { + while let event = try await iterator.next() { + switch event { + case .element(let index, let value): run.elements.append((index, value)) + case .documentCompleted(let summary): run.completions.append(summary) + } + } + } catch let failure as JSONFailure { run.failure = failure } catch { Issue.record("Unexpected error \(error)") } + #expect((try? await iterator.next()) == nil) + return run +} +private func slice(_ text: String, _ span: JSONSourceSpan) -> String { + String(decoding: Array(text.utf8)[Int(span.start)..