From da339360c54e39286a3f43b0a67e6fe27e3ad401 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:14:32 +0100 Subject: [PATCH] Account composition input per document and charge completion Salvage the native composition accounting correction from the historical composition branch (PR #63) onto current main without its performance toolkit. parseDocument retires the preceding input count before the next document, so inputBytes applies per document while the reported total stays cumulative. Completion now charges its own 424-byte counter report and 50 work units on the original ledger. Add the checked reserveExternalLogicalBytes(_:count:overhead:) overload; the two-argument form keeps its signature. Carry the contract test expectations and the two owned cancellation and substitution proofs. Update the composition guide, resource and qualification contracts, discovery counts (JSONIntegrationTests 34 -> 36), CHANGELOG (1.9.0-dev) and the CLI source identity. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 +++ .../Architecture/CompositionQualification.md | 7 ++ Documentation/Architecture/ResourceLimits.md | 10 +++ .../Qualification/TestDiscovery.json | 6 +- Scripts/test_qualification_harness.py | 2 +- .../JSON/Documentation.docc/Composition.md | 39 +++++++++- .../JSON/Limits/JSONCompositionScope.swift | 24 +++++-- Sources/JSON/Limits/JSONWorkMeter.swift | 29 ++++++++ Sources/JSONCLIEngine/CLIBuildIdentity.swift | 2 +- .../CompositionContractTests.swift | 16 ++--- .../CompositionOwnedProofTests.swift | 72 +++++++++++++++++++ 11 files changed, 196 insertions(+), 20 deletions(-) create mode 100644 Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 23a054b..1217597 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased — 1.9.0-dev + +- Composition accounting: apply `inputBytes` per document while keeping reported + input cumulative, and charge completion's own report (424 logical bytes, 50 + work units) on the original ledger. Add the checked + `reserveExternalLogicalBytes(_:count:overhead:)` overload; the two-argument form + is unchanged. Salvaged from the historical composition branch (PR #63) without + its performance toolkit. + ## 1.8.0 — prepared source; publication separate - Add `JSONLocatedTopLevelArraySequence` and `JSONLocatedTopLevelArrayEvent` to diff --git a/Documentation/Architecture/CompositionQualification.md b/Documentation/Architecture/CompositionQualification.md index 47f77bc..ad1d32a 100644 --- a/Documentation/Architecture/CompositionQualification.md +++ b/Documentation/Architecture/CompositionQualification.md @@ -67,6 +67,13 @@ reject missing tests and misclassified resource failures. The reconciled 597 nat Linux root functions, 325 parameter invocations, restored identities and 49 public metric probes retain their existing contracts. +Completion additionally owns a 424-byte counter report and charges 50 work units. +Threshold assertions include that independent buffer; source input capacity is per +document while reported bytes accumulate across all documents. Two root functions +(`jcompOwnedCancellationOccursInsideNativeWork`, +`jcompOriginalHealthAndIdentitySurvivePackageSubstitution`) carry the owned +cancellation and substitution proofs from the historical composition branch (PR #63). + The tests replay exact and one-below work/retention thresholds, compare both strategies on identical input, preserve exact values and scalar keys, and catch late missing-target/cancellation failures before attempting normal completion. diff --git a/Documentation/Architecture/ResourceLimits.md b/Documentation/Architecture/ResourceLimits.md index a7e7581..98f583f 100644 --- a/Documentation/Architecture/ResourceLimits.md +++ b/Documentation/Architecture/ResourceLimits.md @@ -160,6 +160,16 @@ JSONOperationReport initializer permits pairing statistics with another result; its type alone does not establish provenance or adapter completeness. See the [composition guide](../../Sources/JSON/Documentation.docc/Composition.md). +Input capacity is per document: `parseDocument` checkpoints and retires the +preceding input count into the same cumulative report before the next document. +Work, retention, accepted-capacity history and first-failure identity never +restart. Completion admits an additional 424 logical bytes (32 header bytes plus +49 eight-byte counter slots) and 50 work units (49 counter copies and one final +work update) while every result reservation remains live. Overflow, quota failure +and cancellation prevent a completed report even when caught by the adapter. +`reserveExternalLogicalBytes(_:count:overhead:)` checks multiplication, then +addition, before growth; the original two-argument form is unchanged. + ## Optional authored DSL boundary (P1.2) JSONValueDSL and JSONSchemaDSL explicitly accept trusted authored staging. Their diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index 925216c..5ff4dc7 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -6,7 +6,7 @@ "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 36, "JSONPatchTests": 16, - "JSONIntegrationTests": 34, + "JSONIntegrationTests": 36, "JSONHTMLTests": 97, "JSONBridgesTests": 10, "JSONCLITests": 88, @@ -78,7 +78,7 @@ "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 36, "JSONPatchTests": 16, - "JSONIntegrationTests": 34, + "JSONIntegrationTests": 36, "JSONHTMLTests": 81, "JSONBridgesTests": 10, "JSONCLITests": 88, @@ -94,7 +94,7 @@ "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 36, "JSONPatchTests": 16, - "JSONIntegrationTests": 34, + "JSONIntegrationTests": 36, "JSONHTMLTests": 81, "JSONBridgesTests": 10, "JSONNavigationTests": 57, diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py index 4662219..c8d2d66 100644 --- a/Scripts/test_qualification_harness.py +++ b/Scripts/test_qualification_harness.py @@ -168,7 +168,7 @@ def test_composition_discovery_and_capacity_limits_are_not_blanket_successes(sel self.assertEqual(policy[key]['JSONNavigationTests'], 57) self.assertEqual(policy[key]['JSONSchemaExamplesTests'], 98) self.assertEqual(policy[key]['JSONInspectionTests'], 190) - self.assertEqual(policy[key]['JSONIntegrationTests'], 34) + self.assertEqual(policy[key]['JSONIntegrationTests'], 36) for key in ('shared_state_functions', 'portable_shared_state_functions'): self.assertEqual(policy[key]['JSONNavigationTests'], 2) self.assertEqual(policy[key]['JSONSchemaExamplesTests'], 4) diff --git a/Sources/JSON/Documentation.docc/Composition.md b/Sources/JSON/Documentation.docc/Composition.md index d71a518..e22d187 100644 --- a/Sources/JSON/Documentation.docc/Composition.md +++ b/Sources/JSON/Documentation.docc/Composition.md @@ -9,7 +9,7 @@ import Foundation import JSON import JSONSchema -let report = try JSON.withComposition { scope in +let report = try JSON.withComposition { (scope: borrowing JSONCompositionScope) in let document = try scope.parseDocument(Data(#"{"count":9007199254740993}"#.utf8)) try scope.reserveExternalLogicalBytes(64) // Adapter-declared output frame weight. try scope.chargeExternalWork(1) // Before the adapter's corresponding work. @@ -34,11 +34,21 @@ claim about opaque storage. External loops must charge/checkpoint regularly. charges each copied byte and runs the native source-map parser on the same meter. The original caller Data is excluded; the array copy remains conservatively reserved alongside decoded values, source bytes, spans and final output. -Version 1 native weights and internal scratch releases follow the resource +Each document applies `inputBytes` independently. Completed input counts remain +in the cumulative report; work, logical retention, accepted-capacity history and +terminal health never restart. Other native capacities remain conservative across +the scope. Version 1 native weights and internal scratch releases follow the resource contract. Reported peak is this ledger's conservative logical peak, not a sum of independent operation peaks. Final retained bytes include all unreleased reservations, even when the callback returns a smaller result. +Completion reserves 424 logical bytes (32 header bytes plus 49 UInt64 slots) +before allocating the independent report counter buffer. It charges 49 counter +copies plus one final aggregate-work update: 50 work units. The returned result +and all conservative reservations remain live through this final admission. +An adapter must also reserve its result containers before growth; the generic +result type does not reveal arbitrary storage to the library. + All owned helper errors are terminal, including syntax and missing-target errors. Catching one cannot produce a successful report. Scope exit rethrows the original owned failure ahead of any later callback error. Restriction intersects limits @@ -77,3 +87,28 @@ is not a normal rejection, a supported escape or a universal compiler-enforced non-escape proof. Do not infer safety from typecheck success alone. The existing scope, terminal-failure and accounting contracts remain unchanged; runtime invalidation alternatives and later accounting/root proposals need separate review. + + +## Compiler and portability boundary + +Pass the borrowed scope explicitly to adapter helper functions: + +```swift +func visit(_ scope: borrowing JSONCompositionScope) throws { + try scope.chargeExternalWork(1) +} +``` + +Both Apple Swift 6.4.0.34.1 and Swift.org 6.4 reproduce an ownership-verifier +crash when a nested helper captures an inferred closure parameter. Explicitly +annotate `(scope: borrowing JSONCompositionScope)` as above: the nested synchronous +helper compiles, shares the original ledger, and escaping capture receives the +normal borrowed-parameter diagnostic. Ordinary calls also work through explicit +borrowed helper parameters. + +Forming a bound method reference such as `scope.checkpoint` still crashes both +observed compilers, including through an explicit borrowed helper. Call the method +or a helper directly. Reduced reproducers with no JSON imports are preserved on the +historical composition qualification branch (PR #63). These are compiler defects, +not intended negative-test passes; no experimental flags or weaker ownership model +are used. diff --git a/Sources/JSON/Limits/JSONCompositionScope.swift b/Sources/JSON/Limits/JSONCompositionScope.swift index 3ebf9b3..0e0eeee 100644 --- a/Sources/JSON/Limits/JSONCompositionScope.swift +++ b/Sources/JSON/Limits/JSONCompositionScope.swift @@ -44,6 +44,7 @@ public struct JSONCompositionScope: ~Copyable { /// - Throws: Original native failure or cancellation; caught failures stay terminal. public func parseDocument(_ data: Data) throws -> JSONDocument { try withMeter(domain: .parsing) { meter in + try meter.beginCompositionDocument() try meter.require(.inputBytes, UInt64(data.count)) try meter.retain(UInt64(data.count)) var bytes: [UInt8] = [] @@ -124,6 +125,15 @@ public struct JSONCompositionScope: ~Copyable { try withMeter { try $0.work(units) } } + /// Reserves checked adapter bytes before growth, retaining the original API shape. + /// - Parameters: + /// - bytes: Logical weight per unit. + /// - count: Number of units, multiplied with checked arithmetic. + /// - Throws: Terminal quota/overflow failure or cancellation. + public func reserveExternalLogicalBytes(_ bytes: UInt64, count: UInt64 = 1) throws { + try reserveExternalLogicalBytes(bytes, count: count, overhead: 0) + } + /// Reserves adapter-owned logical bytes until scope completion, without refund. /// /// Call before growth. Separate calls count separately even for physical aliases. @@ -132,16 +142,18 @@ public struct JSONCompositionScope: ~Copyable { /// - Parameters: /// - bytes: Logical weight per unit. /// - count: Number of units, multiplied with checked UInt64 arithmetic. + /// - overhead: Additional logical header or bookkeeping bytes, added before growth. /// - Throws: Quota/overflow failure or cancellation, permanently terminal. - public func reserveExternalLogicalBytes(_ bytes: UInt64, count: UInt64 = 1) throws { + public func reserveExternalLogicalBytes(_ bytes: UInt64, count: UInt64 = 1, overhead: UInt64) throws { try withMeter { meter in - let (total, overflow) = bytes.multipliedReportingOverflow(by: count) - guard !overflow else { + let (total, productOverflow) = bytes.multipliedReportingOverflow(by: count) + let (complete, sumOverflow) = total.addingReportingOverflow(overhead) + guard !productOverflow && !sumOverflow else { throw JSONFailure(.resource, code: .counterOverflow, resource: .init( domain: .core, metric: .retainedBytes, limit: meter.limits[.retainedBytes], attemptedUsage: nil, overflow: true)) } - try meter.retain(total) + try meter.retain(complete) } } @@ -163,7 +175,9 @@ public struct JSONCompositionScope: ~Copyable { do { let result = try body(scope) try original.check(scope.state.context.meter) - return .init(result: result, statistics: scope.state.context.meter.statistics()) + let statistics = try scope.withMeter { try $0.compositionStatistics() } + try original.check(scope.state.context.meter) + return .init(result: result, statistics: statistics) } catch { // A caller error cannot conceal an earlier native terminal failure or substitution. try original.check(scope.state.context.meter) diff --git a/Sources/JSON/Limits/JSONWorkMeter.swift b/Sources/JSON/Limits/JSONWorkMeter.swift index 0f5a1f2..73b19cd 100644 --- a/Sources/JSON/Limits/JSONWorkMeter.swift +++ b/Sources/JSON/Limits/JSONWorkMeter.swift @@ -206,6 +206,35 @@ package struct JSONWorkMeter: ~Copyable { guard bytes <= used(.annotationBytes) else { throw JSONFailure(.configuration, code: .invalidLimits) } counters[JSONResourceMetric.annotationBytes.index] -= bytes } + /// Input is a per-document capacity. Retire its charged count into the same + /// cumulative report; never reset work, retention, accepted history or health. + package mutating func beginCompositionDocument() throws { + try checkpoint() + try retire(.inputBytes) + } + /// Composition-only completion owns a separate 49-slot report through handoff. + /// Existing report entry points retain their original accounting semantics. + package mutating func compositionStatistics() throws -> JSONOperationStatistics { + // Version 1 composition receipt: 32 logical header bytes + 49 UInt64 slots. + try retain(424) + var report: [UInt64] = [] + report.reserveCapacity(JSONResourceMetric.allCases.count) + for metric in JSONResourceMetric.allCases { + try work() + let (total, overflow) = counters[metric.index].addingReportingOverflow(retired[metric.index]) + guard !overflow else { throw violation(metric, nil, overflow: true) } + report.append(total) + } + // The aggregate-work slot may precede the last copied slot. Charge and + // publish the final count only after every copied counter is accounted. + try work() + let (work, overflow) = counters[JSONResourceMetric.workUnits.index] + .addingReportingOverflow(retired[JSONResourceMetric.workUnits.index]) + guard !overflow else { throw violation(.workUnits, nil, overflow: true) } + report[JSONResourceMetric.workUnits.index] = work + try checkpoint() + return .init(domain: domain, counters: report, peak: peak) + } package func statistics() -> JSONOperationStatistics { var report = counters for metric in JSONResourceMetric.allCases { report[metric.index] += retired[metric.index] } diff --git a/Sources/JSONCLIEngine/CLIBuildIdentity.swift b/Sources/JSONCLIEngine/CLIBuildIdentity.swift index d207128..76cf076 100644 --- a/Sources/JSONCLIEngine/CLIBuildIdentity.swift +++ b/Sources/JSONCLIEngine/CLIBuildIdentity.swift @@ -2,6 +2,6 @@ // Generated by Scripts/generate_cli_identity.py; excludes this file. // SHA-256 of the ordered Package.swift + Sources content manifest, not a Git commit. package enum CLIBuildIdentity { - package static let revision = "sha256:ff976484187552cfd967cc4a750bd1f32f63bf37137e01b7343d4152a385273a" + package static let revision = "sha256:266a412db5d592d449264797e72a2d652d636eb431ef62d90a389e85e33d9bde" package static let kind = "source-sha256" } diff --git a/Tests/JSONIntegrationTests/CompositionContractTests.swift b/Tests/JSONIntegrationTests/CompositionContractTests.swift index c650339..57c04a5 100644 --- a/Tests/JSONIntegrationTests/CompositionContractTests.swift +++ b/Tests/JSONIntegrationTests/CompositionContractTests.swift @@ -91,11 +91,11 @@ private func composedImport(limits: JSONLimits = .balanced, extra: UInt64 = 0) t try scope.reserveExternalLogicalBytes(64) return 0 } - #expect(two.statistics.peakLogicalRetainedBytes == 128) - #expect(two.statistics[.retainedBytes] == 128) - #expect(one.statistics.peakLogicalRetainedBytes == 64) + #expect(two.statistics.peakLogicalRetainedBytes == 128 + 424) + #expect(two.statistics[.retainedBytes] == 128 + 424) + #expect(one.statistics.peakLogicalRetainedBytes == 64 + 424) var belowOverlap = JSONLimits.balanced - belowOverlap[.retainedBytes] = 127 + belowOverlap[.retainedBytes] = 127 + 424 #expect(throws: JSONFailure.self) { try JSON.withComposition(limits: belowOverlap) { scope in try scope.reserveExternalLogicalBytes(64) @@ -103,7 +103,7 @@ private func composedImport(limits: JSONLimits = .balanced, extra: UInt64 = 0) t return 0 } } - #expect(try JSON.withComposition { try $0.admitTrusted(.null) }.statistics[.retainedBytes] == 32) + #expect(try JSON.withComposition { try $0.admitTrusted(.null) }.statistics[.retainedBytes] == 32 + 424) // Independent stage peaks are not a composition oracle: the parser's scratch // peak and a later URI peak need not be simultaneously live. let parse = try JSON.withComposition { try $0.parseDocument(compositionInput) } @@ -178,7 +178,7 @@ private func composedImport(limits: JSONLimits = .balanced, extra: UInt64 = 0) t _ = try scope.admitTrusted(alias) return alias } - #expect(two.statistics[.retainedBytes] == one.statistics[.retainedBytes] * 2) + #expect(two.statistics[.retainedBytes] == (one.statistics[.retainedBytes] - 424) * 2 + 424) var limits = JSONLimits.balanced limits[.compiledPlanBytes] = 0 limits[.compileWorkUnits] = 0 @@ -303,10 +303,10 @@ private func composedImport(limits: JSONLimits = .balanced, extra: UInt64 = 0) t try inner.chargeExternalWork(5) return 0 } - #expect(inner.statistics[.workUnits] == 5) + #expect(inner.statistics[.workUnits] == 5 + 50) return 0 } - #expect(report.statistics[.workUnits] == 3) + #expect(report.statistics[.workUnits] == 3 + 50) // Substitution of the borrowed facade is rejected by the external compiler // fixture Substitute.swift.txt, including swap/use/restore. Independent // nested invocations are explicitly outside the outer report's claim. diff --git a/Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift b/Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift new file mode 100644 index 0000000..176f348 --- /dev/null +++ b/Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Synchronization +import Testing +import JSON +import JSONSchema + +// Native-only rendezvous is supplementary to the unrelated public package. It +// uses the existing invocation-owned observer; it is not an accounting oracle. +@Test func jcompOwnedCancellationOccursInsideNativeWork() async throws { + for stage in 0..<7 { + let seen = Mutex(nil) + let task = Task.detached { + var limits = JSONLimits.offline; limits[.cancellationQuantum] = 1 + let threshold: UInt64 = stage == 1 ? 4200 : stage == 4 ? 6 : 32 + return try JSON.withComposition(limits: limits) { scope in + try scope.withMeter { meter in + meter.checkpointObserver = { work in + if work >= threshold { + seen.withLock { if $0 == nil { $0 = work } } + withUnsafeCurrentTask { $0?.cancel() } + } + } + } + do { + switch stage { + case 0, 1: + _ = try scope.parseDocument(Data(("\"" + String(repeating: "a", count: 4096) + "\"").utf8)) + case 2: _ = try scope.pointer(parsing: "/" + String(repeating: "a", count: 4096)) + case 3: _ = try scope.uri("https://example.invalid/" + String(repeating: "a", count: 4096)) + case 4: try scope.chargeExternalWork(5) // Cancellation must occur in report copying. + case 5: + for _ in 0..<128 { try scope.chargeExternalWork(1) } + default: + let path = try scope.pointer(tokens: []) + _ = try scope.resolve(path, in: .array(Array(repeating: .null, count: 256))) + } + } catch is CancellationError { } + return 1 + } + } + do { _ = try await task.value; Issue.record("caught cancellation completed at stage \(stage)") } + catch is CancellationError { } + #expect(seen.withLock { $0 != nil }) + if stage == 1 { #expect(seen.withLock { ($0 ?? 0) > 4098 }) } + } +} + +@Test func jcompOriginalHealthAndIdentitySurvivePackageSubstitution() throws { + for stage in 0..<3 { + do { + _ = try JSON.withComposition { scope in + try scope.withMeter { meter in + if stage == 1 { + do { try meter.work(UInt64.max) } catch { } + } else if stage == 2 { + do { try meter.retain(UInt64.max) } catch { } + } + // Impossible through public API; exercise the original + // continuation's identity and first-cause check directly. + meter = try JSONWorkMeter(limits: .balanced, domain: .core) + } + return 1 + } + Issue.record("substituted native ledger completed") + } catch let failure as JSONFailure { + if stage == 0 { #expect(failure.category == .configuration && failure.code == .invalidOperation) } + if stage == 1 { #expect(failure.resource?.metric == .workUnits) } + if stage == 2 { #expect(failure.resource?.metric == .retainedBytes) } + } + } +}