diff --git a/CHANGELOG.md b/CHANGELOG.md index 23a054b..1217597 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased — 1.9.0-dev + +- Composition accounting: apply `inputBytes` per document while keeping reported + input cumulative, and charge completion's own report (424 logical bytes, 50 + work units) on the original ledger. Add the checked + `reserveExternalLogicalBytes(_:count:overhead:)` overload; the two-argument form + is unchanged. Salvaged from the historical composition branch (PR #63) without + its performance toolkit. + ## 1.8.0 — prepared source; publication separate - Add `JSONLocatedTopLevelArraySequence` and `JSONLocatedTopLevelArrayEvent` to diff --git a/Documentation/Architecture/CompositionQualification.md b/Documentation/Architecture/CompositionQualification.md index 47f77bc..ad1d32a 100644 --- a/Documentation/Architecture/CompositionQualification.md +++ b/Documentation/Architecture/CompositionQualification.md @@ -67,6 +67,13 @@ reject missing tests and misclassified resource failures. The reconciled 597 nat Linux root functions, 325 parameter invocations, restored identities and 49 public metric probes retain their existing contracts. +Completion additionally owns a 424-byte counter report and charges 50 work units. +Threshold assertions include that independent buffer; source input capacity is per +document while reported bytes accumulate across all documents. Two root functions +(`jcompOwnedCancellationOccursInsideNativeWork`, +`jcompOriginalHealthAndIdentitySurvivePackageSubstitution`) carry the owned +cancellation and substitution proofs from the historical composition branch (PR #63). + The tests replay exact and one-below work/retention thresholds, compare both strategies on identical input, preserve exact values and scalar keys, and catch late missing-target/cancellation failures before attempting normal completion. diff --git a/Documentation/Architecture/ResourceLimits.md b/Documentation/Architecture/ResourceLimits.md index a7e7581..98f583f 100644 --- a/Documentation/Architecture/ResourceLimits.md +++ b/Documentation/Architecture/ResourceLimits.md @@ -160,6 +160,16 @@ JSONOperationReport initializer permits pairing statistics with another result; its type alone does not establish provenance or adapter completeness. See the [composition guide](../../Sources/JSON/Documentation.docc/Composition.md). +Input capacity is per document: `parseDocument` checkpoints and retires the +preceding input count into the same cumulative report before the next document. +Work, retention, accepted-capacity history and first-failure identity never +restart. Completion admits an additional 424 logical bytes (32 header bytes plus +49 eight-byte counter slots) and 50 work units (49 counter copies and one final +work update) while every result reservation remains live. Overflow, quota failure +and cancellation prevent a completed report even when caught by the adapter. +`reserveExternalLogicalBytes(_:count:overhead:)` checks multiplication, then +addition, before growth; the original two-argument form is unchanged. + ## Optional authored DSL boundary (P1.2) JSONValueDSL and JSONSchemaDSL explicitly accept trusted authored staging. Their diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index 925216c..5ff4dc7 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -6,7 +6,7 @@ "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 36, "JSONPatchTests": 16, - "JSONIntegrationTests": 34, + "JSONIntegrationTests": 36, "JSONHTMLTests": 97, "JSONBridgesTests": 10, "JSONCLITests": 88, @@ -78,7 +78,7 @@ "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 36, "JSONPatchTests": 16, - "JSONIntegrationTests": 34, + "JSONIntegrationTests": 36, "JSONHTMLTests": 81, "JSONBridgesTests": 10, "JSONCLITests": 88, @@ -94,7 +94,7 @@ "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 36, "JSONPatchTests": 16, - "JSONIntegrationTests": 34, + "JSONIntegrationTests": 36, "JSONHTMLTests": 81, "JSONBridgesTests": 10, "JSONNavigationTests": 57, diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py index 4662219..c8d2d66 100644 --- a/Scripts/test_qualification_harness.py +++ b/Scripts/test_qualification_harness.py @@ -168,7 +168,7 @@ def test_composition_discovery_and_capacity_limits_are_not_blanket_successes(sel self.assertEqual(policy[key]['JSONNavigationTests'], 57) self.assertEqual(policy[key]['JSONSchemaExamplesTests'], 98) self.assertEqual(policy[key]['JSONInspectionTests'], 190) - self.assertEqual(policy[key]['JSONIntegrationTests'], 34) + self.assertEqual(policy[key]['JSONIntegrationTests'], 36) for key in ('shared_state_functions', 'portable_shared_state_functions'): self.assertEqual(policy[key]['JSONNavigationTests'], 2) self.assertEqual(policy[key]['JSONSchemaExamplesTests'], 4) diff --git a/Sources/JSON/Documentation.docc/Composition.md b/Sources/JSON/Documentation.docc/Composition.md index d71a518..e22d187 100644 --- a/Sources/JSON/Documentation.docc/Composition.md +++ b/Sources/JSON/Documentation.docc/Composition.md @@ -9,7 +9,7 @@ import Foundation import JSON import JSONSchema -let report = try JSON.withComposition { scope in +let report = try JSON.withComposition { (scope: borrowing JSONCompositionScope) in let document = try scope.parseDocument(Data(#"{"count":9007199254740993}"#.utf8)) try scope.reserveExternalLogicalBytes(64) // Adapter-declared output frame weight. try scope.chargeExternalWork(1) // Before the adapter's corresponding work. @@ -34,11 +34,21 @@ claim about opaque storage. External loops must charge/checkpoint regularly. charges each copied byte and runs the native source-map parser on the same meter. The original caller Data is excluded; the array copy remains conservatively reserved alongside decoded values, source bytes, spans and final output. -Version 1 native weights and internal scratch releases follow the resource +Each document applies `inputBytes` independently. Completed input counts remain +in the cumulative report; work, logical retention, accepted-capacity history and +terminal health never restart. Other native capacities remain conservative across +the scope. Version 1 native weights and internal scratch releases follow the resource contract. Reported peak is this ledger's conservative logical peak, not a sum of independent operation peaks. Final retained bytes include all unreleased reservations, even when the callback returns a smaller result. +Completion reserves 424 logical bytes (32 header bytes plus 49 UInt64 slots) +before allocating the independent report counter buffer. It charges 49 counter +copies plus one final aggregate-work update: 50 work units. The returned result +and all conservative reservations remain live through this final admission. +An adapter must also reserve its result containers before growth; the generic +result type does not reveal arbitrary storage to the library. + All owned helper errors are terminal, including syntax and missing-target errors. Catching one cannot produce a successful report. Scope exit rethrows the original owned failure ahead of any later callback error. Restriction intersects limits @@ -77,3 +87,28 @@ is not a normal rejection, a supported escape or a universal compiler-enforced non-escape proof. Do not infer safety from typecheck success alone. The existing scope, terminal-failure and accounting contracts remain unchanged; runtime invalidation alternatives and later accounting/root proposals need separate review. + + +## Compiler and portability boundary + +Pass the borrowed scope explicitly to adapter helper functions: + +```swift +func visit(_ scope: borrowing JSONCompositionScope) throws { + try scope.chargeExternalWork(1) +} +``` + +Both Apple Swift 6.4.0.34.1 and Swift.org 6.4 reproduce an ownership-verifier +crash when a nested helper captures an inferred closure parameter. Explicitly +annotate `(scope: borrowing JSONCompositionScope)` as above: the nested synchronous +helper compiles, shares the original ledger, and escaping capture receives the +normal borrowed-parameter diagnostic. Ordinary calls also work through explicit +borrowed helper parameters. + +Forming a bound method reference such as `scope.checkpoint` still crashes both +observed compilers, including through an explicit borrowed helper. Call the method +or a helper directly. Reduced reproducers with no JSON imports are preserved on the +historical composition qualification branch (PR #63). These are compiler defects, +not intended negative-test passes; no experimental flags or weaker ownership model +are used. diff --git a/Sources/JSON/Limits/JSONCompositionScope.swift b/Sources/JSON/Limits/JSONCompositionScope.swift index 3ebf9b3..0e0eeee 100644 --- a/Sources/JSON/Limits/JSONCompositionScope.swift +++ b/Sources/JSON/Limits/JSONCompositionScope.swift @@ -44,6 +44,7 @@ public struct JSONCompositionScope: ~Copyable { /// - Throws: Original native failure or cancellation; caught failures stay terminal. public func parseDocument(_ data: Data) throws -> JSONDocument { try withMeter(domain: .parsing) { meter in + try meter.beginCompositionDocument() try meter.require(.inputBytes, UInt64(data.count)) try meter.retain(UInt64(data.count)) var bytes: [UInt8] = [] @@ -124,6 +125,15 @@ public struct JSONCompositionScope: ~Copyable { try withMeter { try $0.work(units) } } + /// Reserves checked adapter bytes before growth, retaining the original API shape. + /// - Parameters: + /// - bytes: Logical weight per unit. + /// - count: Number of units, multiplied with checked arithmetic. + /// - Throws: Terminal quota/overflow failure or cancellation. + public func reserveExternalLogicalBytes(_ bytes: UInt64, count: UInt64 = 1) throws { + try reserveExternalLogicalBytes(bytes, count: count, overhead: 0) + } + /// Reserves adapter-owned logical bytes until scope completion, without refund. /// /// Call before growth. Separate calls count separately even for physical aliases. @@ -132,16 +142,18 @@ public struct JSONCompositionScope: ~Copyable { /// - Parameters: /// - bytes: Logical weight per unit. /// - count: Number of units, multiplied with checked UInt64 arithmetic. + /// - overhead: Additional logical header or bookkeeping bytes, added before growth. /// - Throws: Quota/overflow failure or cancellation, permanently terminal. - public func reserveExternalLogicalBytes(_ bytes: UInt64, count: UInt64 = 1) throws { + public func reserveExternalLogicalBytes(_ bytes: UInt64, count: UInt64 = 1, overhead: UInt64) throws { try withMeter { meter in - let (total, overflow) = bytes.multipliedReportingOverflow(by: count) - guard !overflow else { + let (total, productOverflow) = bytes.multipliedReportingOverflow(by: count) + let (complete, sumOverflow) = total.addingReportingOverflow(overhead) + guard !productOverflow && !sumOverflow else { throw JSONFailure(.resource, code: .counterOverflow, resource: .init( domain: .core, metric: .retainedBytes, limit: meter.limits[.retainedBytes], attemptedUsage: nil, overflow: true)) } - try meter.retain(total) + try meter.retain(complete) } } @@ -163,7 +175,9 @@ public struct JSONCompositionScope: ~Copyable { do { let result = try body(scope) try original.check(scope.state.context.meter) - return .init(result: result, statistics: scope.state.context.meter.statistics()) + let statistics = try scope.withMeter { try $0.compositionStatistics() } + try original.check(scope.state.context.meter) + return .init(result: result, statistics: statistics) } catch { // A caller error cannot conceal an earlier native terminal failure or substitution. try original.check(scope.state.context.meter) diff --git a/Sources/JSON/Limits/JSONWorkMeter.swift b/Sources/JSON/Limits/JSONWorkMeter.swift index 0f5a1f2..73b19cd 100644 --- a/Sources/JSON/Limits/JSONWorkMeter.swift +++ b/Sources/JSON/Limits/JSONWorkMeter.swift @@ -206,6 +206,35 @@ package struct JSONWorkMeter: ~Copyable { guard bytes <= used(.annotationBytes) else { throw JSONFailure(.configuration, code: .invalidLimits) } counters[JSONResourceMetric.annotationBytes.index] -= bytes } + /// Input is a per-document capacity. Retire its charged count into the same + /// cumulative report; never reset work, retention, accepted history or health. + package mutating func beginCompositionDocument() throws { + try checkpoint() + try retire(.inputBytes) + } + /// Composition-only completion owns a separate 49-slot report through handoff. + /// Existing report entry points retain their original accounting semantics. + package mutating func compositionStatistics() throws -> JSONOperationStatistics { + // Version 1 composition receipt: 32 logical header bytes + 49 UInt64 slots. + try retain(424) + var report: [UInt64] = [] + report.reserveCapacity(JSONResourceMetric.allCases.count) + for metric in JSONResourceMetric.allCases { + try work() + let (total, overflow) = counters[metric.index].addingReportingOverflow(retired[metric.index]) + guard !overflow else { throw violation(metric, nil, overflow: true) } + report.append(total) + } + // The aggregate-work slot may precede the last copied slot. Charge and + // publish the final count only after every copied counter is accounted. + try work() + let (work, overflow) = counters[JSONResourceMetric.workUnits.index] + .addingReportingOverflow(retired[JSONResourceMetric.workUnits.index]) + guard !overflow else { throw violation(.workUnits, nil, overflow: true) } + report[JSONResourceMetric.workUnits.index] = work + try checkpoint() + return .init(domain: domain, counters: report, peak: peak) + } package func statistics() -> JSONOperationStatistics { var report = counters for metric in JSONResourceMetric.allCases { report[metric.index] += retired[metric.index] } diff --git a/Sources/JSONCLIEngine/CLIBuildIdentity.swift b/Sources/JSONCLIEngine/CLIBuildIdentity.swift index d207128..76cf076 100644 --- a/Sources/JSONCLIEngine/CLIBuildIdentity.swift +++ b/Sources/JSONCLIEngine/CLIBuildIdentity.swift @@ -2,6 +2,6 @@ // Generated by Scripts/generate_cli_identity.py; excludes this file. // SHA-256 of the ordered Package.swift + Sources content manifest, not a Git commit. package enum CLIBuildIdentity { - package static let revision = "sha256:ff976484187552cfd967cc4a750bd1f32f63bf37137e01b7343d4152a385273a" + package static let revision = "sha256:266a412db5d592d449264797e72a2d652d636eb431ef62d90a389e85e33d9bde" package static let kind = "source-sha256" } diff --git a/Tests/JSONIntegrationTests/CompositionContractTests.swift b/Tests/JSONIntegrationTests/CompositionContractTests.swift index c650339..57c04a5 100644 --- a/Tests/JSONIntegrationTests/CompositionContractTests.swift +++ b/Tests/JSONIntegrationTests/CompositionContractTests.swift @@ -91,11 +91,11 @@ private func composedImport(limits: JSONLimits = .balanced, extra: UInt64 = 0) t try scope.reserveExternalLogicalBytes(64) return 0 } - #expect(two.statistics.peakLogicalRetainedBytes == 128) - #expect(two.statistics[.retainedBytes] == 128) - #expect(one.statistics.peakLogicalRetainedBytes == 64) + #expect(two.statistics.peakLogicalRetainedBytes == 128 + 424) + #expect(two.statistics[.retainedBytes] == 128 + 424) + #expect(one.statistics.peakLogicalRetainedBytes == 64 + 424) var belowOverlap = JSONLimits.balanced - belowOverlap[.retainedBytes] = 127 + belowOverlap[.retainedBytes] = 127 + 424 #expect(throws: JSONFailure.self) { try JSON.withComposition(limits: belowOverlap) { scope in try scope.reserveExternalLogicalBytes(64) @@ -103,7 +103,7 @@ private func composedImport(limits: JSONLimits = .balanced, extra: UInt64 = 0) t return 0 } } - #expect(try JSON.withComposition { try $0.admitTrusted(.null) }.statistics[.retainedBytes] == 32) + #expect(try JSON.withComposition { try $0.admitTrusted(.null) }.statistics[.retainedBytes] == 32 + 424) // Independent stage peaks are not a composition oracle: the parser's scratch // peak and a later URI peak need not be simultaneously live. let parse = try JSON.withComposition { try $0.parseDocument(compositionInput) } @@ -178,7 +178,7 @@ private func composedImport(limits: JSONLimits = .balanced, extra: UInt64 = 0) t _ = try scope.admitTrusted(alias) return alias } - #expect(two.statistics[.retainedBytes] == one.statistics[.retainedBytes] * 2) + #expect(two.statistics[.retainedBytes] == (one.statistics[.retainedBytes] - 424) * 2 + 424) var limits = JSONLimits.balanced limits[.compiledPlanBytes] = 0 limits[.compileWorkUnits] = 0 @@ -303,10 +303,10 @@ private func composedImport(limits: JSONLimits = .balanced, extra: UInt64 = 0) t try inner.chargeExternalWork(5) return 0 } - #expect(inner.statistics[.workUnits] == 5) + #expect(inner.statistics[.workUnits] == 5 + 50) return 0 } - #expect(report.statistics[.workUnits] == 3) + #expect(report.statistics[.workUnits] == 3 + 50) // Substitution of the borrowed facade is rejected by the external compiler // fixture Substitute.swift.txt, including swap/use/restore. Independent // nested invocations are explicitly outside the outer report's claim. diff --git a/Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift b/Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift new file mode 100644 index 0000000..176f348 --- /dev/null +++ b/Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Synchronization +import Testing +import JSON +import JSONSchema + +// Native-only rendezvous is supplementary to the unrelated public package. It +// uses the existing invocation-owned observer; it is not an accounting oracle. +@Test func jcompOwnedCancellationOccursInsideNativeWork() async throws { + for stage in 0..<7 { + let seen = Mutex(nil) + let task = Task.detached { + var limits = JSONLimits.offline; limits[.cancellationQuantum] = 1 + let threshold: UInt64 = stage == 1 ? 4200 : stage == 4 ? 6 : 32 + return try JSON.withComposition(limits: limits) { scope in + try scope.withMeter { meter in + meter.checkpointObserver = { work in + if work >= threshold { + seen.withLock { if $0 == nil { $0 = work } } + withUnsafeCurrentTask { $0?.cancel() } + } + } + } + do { + switch stage { + case 0, 1: + _ = try scope.parseDocument(Data(("\"" + String(repeating: "a", count: 4096) + "\"").utf8)) + case 2: _ = try scope.pointer(parsing: "/" + String(repeating: "a", count: 4096)) + case 3: _ = try scope.uri("https://example.invalid/" + String(repeating: "a", count: 4096)) + case 4: try scope.chargeExternalWork(5) // Cancellation must occur in report copying. + case 5: + for _ in 0..<128 { try scope.chargeExternalWork(1) } + default: + let path = try scope.pointer(tokens: []) + _ = try scope.resolve(path, in: .array(Array(repeating: .null, count: 256))) + } + } catch is CancellationError { } + return 1 + } + } + do { _ = try await task.value; Issue.record("caught cancellation completed at stage \(stage)") } + catch is CancellationError { } + #expect(seen.withLock { $0 != nil }) + if stage == 1 { #expect(seen.withLock { ($0 ?? 0) > 4098 }) } + } +} + +@Test func jcompOriginalHealthAndIdentitySurvivePackageSubstitution() throws { + for stage in 0..<3 { + do { + _ = try JSON.withComposition { scope in + try scope.withMeter { meter in + if stage == 1 { + do { try meter.work(UInt64.max) } catch { } + } else if stage == 2 { + do { try meter.retain(UInt64.max) } catch { } + } + // Impossible through public API; exercise the original + // continuation's identity and first-cause check directly. + meter = try JSONWorkMeter(limits: .balanced, domain: .core) + } + return 1 + } + Issue.record("substituted native ledger completed") + } catch let failure as JSONFailure { + if stage == 0 { #expect(failure.category == .configuration && failure.code == .invalidOperation) } + if stage == 1 { #expect(failure.resource?.metric == .workUnits) } + if stage == 2 { #expect(failure.resource?.metric == .retainedBytes) } + } + } +}