From ecbae849884ec8a3736cc42c4c1649d6ada26000 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 11:57:17 +0000 Subject: [PATCH 1/9] test(oracles): law constructed independence in the framework self-oracle An appended pair, two synthetic properties alike except that the second's constructed list is the first's with one history appended, must run the same generated histories, in the same order, at every fixed seed and at the seeded run's random seed, and the same constructed histories of the shorter list. RED on main: the campaign's oneof weights the constructed histories by their count, so appending one moves the generated draws. Co-authored-by: Isaac --- .../tests/oracle-property-self-oracle.test.ts | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index aa4e6046..55599fda 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -38,6 +38,16 @@ * fast-check itself, never through the framework. A unit that fails ran that stream up to its * first failure, then its shrinks. The random units of a property run the one seed it logged: the * seed of the replay environment in a seeded run, and a drawn seed in a seedless run. + * - `constructed independence` (maintainer decision, 2026-11-16, review of #121, F2): at a fixed + * seed, the generated histories a campaign runs do not depend on its constructed list, its + * length or its contents, and each constructed history runs as declared, independently of the + * generated draw. So appending a constructed history keeps every fixed seed's histories, which + * the decisions register's generator rules assume. Judged on an appended pair: two properties + * alike except that the second's constructed list is the first's with one history appended. At + * each fixed seed, and at the seeded run's random seed, the `generated` unit and the unit of each + * constructed history of the shorter list run the same histories in the same order in both, up + * to the shorter list's first failure, and the appended history's own unit runs. The law + * compares the two properties' own logs, so it holds without the model's streams. * - `verdict`: a test fails with a violation of law L exactly when its first * failing history violates L, whatever values the history holds (a bigint, * a Map, NaN, or -0 too). Within one history a driver error decides @@ -2860,8 +2870,12 @@ const timeoutReceivingKinds = [ const timeoutReceivingStates = timeoutReceivingKinds.map( (kind) => `timeout-input-${kind}` as const, ) +// Law `constructed independence`: the required states of an appended pair. +const appendedStates = [`appended-to-empty-list`, `appended-to-listed`] as const const frameworkStates = [ ...timeoutReceivingStates, + ...appendedStates, + `appended-pair-whole-stream`, `timeout-presentation-equivalence`, `timeout-continuation-cut`, // A slow spec that fails to load, in a plan that presents timeouts: the @@ -3905,6 +3919,41 @@ function frameworkReach( ) ) reached.add(`timeout-continuation-cut`) + // Law `constructed independence` compared a pair: both specs loaded, the + // shorter list's `generated` unit ran at its first fixed seed, and the + // appended history's unit ran. The shorter list is empty or holds some + // constructed histories; a `generated` unit that ran its whole stream + // without a failure compares every history it drew. + for (const pair of appendedPairs(plan, expected)) { + const [seed] = fixedSeedsOf(pair.listed) + const ran = (spec: SyntheticSpec, unit: string) => + (run?.log ?? []) + .filter((entry) => entry.test === `${spec.id} > ${unit}`) + .map((entry) => entry.history) + const generated = ran(pair.listed, fixedUnit(seed!, `generated`)) + if ( + !run?.files[pair.listed.id]?.loaded || + !run.files[pair.appended.id]?.loaded || + generated.length === 0 || + ran( + pair.appended, + fixedUnit(seed!, pair.appended.constructed.at(-1)!.name), + ).length === 0 + ) + continue + reached.add( + pair.listed.constructed.length === 0 + ? `appended-to-empty-list` + : `appended-to-listed`, + ) + if ( + generated.every( + (history) => + !failsUnit(pair.listed, fixedUnit(seed!, `generated`), history), + ) + ) + reached.add(`appended-pair-whole-stream`) + } return reached } @@ -6005,6 +6054,84 @@ const comparedRun = ( } } +// Law `constructed independence`. Two specs of one plan are an appended pair +// when the second is the first with one constructed history appended to its +// list: the same seed, runs, and every other part, whatever the appended +// history holds and whichever states it adds to the declaration. The relation +// is structural, so it holds for any suite that draws such a pair, not only +// for the constructed histories that build one. A slow spec is left out, as +// law `units` leaves it out: its driver runs on after its test timed out. +type AppendedPair = { listed: SyntheticSpec; appended: SyntheticSpec } +const withoutList = ({ + id: _id, + constructed: _constructed, + states: _states, + ...rest +}: SyntheticSpec) => JSON.stringify(rest) +function appendedPairs( + plan: Plan, + expected: ReturnType, +): Array { + const runs = plan.specs.filter( + (spec) => + spec.source.kind === `generated` && + !spec.slow && + expected.get(spec.id)!.loads, + ) + return runs.flatMap((listed) => + runs + .filter( + (appended) => + appended.constructed.length === listed.constructed.length + 1 && + JSON.stringify(appended.constructed.slice(0, -1)) === + JSON.stringify(listed.constructed) && + withoutList(appended) === withoutList(listed), + ) + .map((appended) => ({ listed, appended })), + ) +} +// What an appended pair ran, judged without the model's streams. At each +// fixed seed, and at the seeded child run's random seed, the `generated` unit +// and the unit of each constructed history of the shorter list ran at least +// one history in both specs, and the same histories in the same order, up to +// the shorter list's first failure (shrinks are not compared). The appended +// history's own unit ran too. Returns the first difference, or undefined. +function appendedDifference( + { listed, appended }: AppendedPair, + log: ReadonlyArray, +): { unit: string; listed: unknown; appended: unknown } | undefined { + const ran = (spec: SyntheticSpec, unit: string) => + log + .filter((entry) => entry.test === `${spec.id} > ${unit}`) + .map((entry) => entry.history) + const names = unitNames(listed) + const shared = [ + ...fixedSeedsOf(listed).flatMap((seed) => + names.map((name) => fixedUnit(seed, name)), + ), + ...names.map(randomUnit), + ] + for (const unit of shared) { + const before = ran(listed, unit) + const first = before.findIndex((history) => + failsUnit(listed, unit, history), + ) + const cut = first < 0 ? before.length : first + 1 + const after = ran(appended, unit) + const compared = first < 0 ? after : after.slice(0, cut) + if ( + before.length === 0 || + JSON.stringify(before.slice(0, cut)) !== JSON.stringify(compared) + ) + return { unit, listed: before.slice(0, cut), appended: compared } + } + const own = appended.constructed.at(-1)!.name + for (const seed of fixedSeedsOf(appended)) + if (ran(appended, fixedUnit(seed, own)).length === 0) + return { unit: fixedUnit(seed, own), listed: null, appended: [] } + return undefined +} + // Model law `replay-config`: the first error of the drawn replay // environment, in the order the law states, or none. function environmentError(plan: Plan): string | undefined { @@ -6576,6 +6703,19 @@ function checkSuite( `host RNG scope`, ) } + // Law `constructed independence`: appending a constructed history changes + // no history that the shorter list's campaigns ran. The comparison is + // between the two specs' own logs, so it needs no model stream. + for (const pair of appendedPairs(plan, expected)) { + const difference = appendedDifference(pair, run.log) + if (difference) + return violation( + `run`, + `${pair.appended.id} "${difference.unit}" ran ${JSON.stringify(difference.appended)} with one constructed history appended to the list of ${pair.listed.id}, which ran ${JSON.stringify(difference.listed)}`, + `constructed independence`, + { actual: difference.appended, expected: difference.listed }, + ) + } for (const spec of plan.specs) { const owed = expected.get(spec.id)! const seen = run.files[spec.id]! @@ -8143,6 +8283,54 @@ const unitWitnessSuite = ( environment: null, ...options, }) +// Law `constructed independence`: a pair of drafts, the second the first with +// one constructed history appended. The first draft's seed, runs, list, and +// naming come from the generated suite, so random seeds vary them; a drawn +// list is kept to the ordinary items, so each of its histories enters the +// state of its first item. The appended history's prefix comes from the +// suite's first fixed history, and it enters a state in some suites only, so +// the pair's declared states differ in those. An enumerated first draft +// gives the pair a generated source of its own. +const ordinaryPrefix = (history: SyntheticHistory | undefined, or: number) => { + const items = (history ?? []).slice(0, 2).map((entry) => Math.abs(entry) % 3) + return items.length > 0 ? items : [or] +} +const appendedPairSuite = (generated: Suite, listed: boolean): Suite => { + const base = generated.drafts[0]! + const tail = base.clock ? 1 : 2 + const constructed = listed + ? (base.constructed.length > 0 + ? base.constructed + : [{ prefix: [tail], enters: [] }] + ).map(({ prefix }) => { + const items = ordinaryPrefix(prefix, tail) + return { prefix: items, enters: [`saw-${items[0]}`] } + }) + : [] + const first = generatedDraft({ + source: + base.source.kind === `generated` + ? base.source + : { kind: `generated`, runs: 3, seed: 20 + tail }, + constructed, + constructedNames: + base.constructedNames === `prefixed` ? `prefixed` : `indexed`, + secondSeed: base.secondSeed, + isolatedPlans: base.isolatedPlans, + reachVia: base.reachVia, + }) + const prefix = ordinaryPrefix(base.fixed[0]?.history, tail) + return unitWitnessSuite([ + first, + { + ...first, + constructed: [ + ...constructed, + { prefix, enters: base.isolatedPlans ? [`saw-${prefix[0]}`] : [] }, + ], + }, + ]) +} const mainUnitRequiredConstructed: Array<{ name: string enters: ReadonlyArray<(typeof mainUnitRequiredStates)[number]> @@ -8334,9 +8522,11 @@ oracleProperty({ `bound-calibration`, `bound-calibration-registration-error`, `bound-constructed-driver`, + ...appendedStates, ], measured: frameworkStates.filter( (state) => + !appendedStates.some((required) => required === state) && state !== `timeout-presentation-equivalence` && state !== `timeout-continuation-cut` && !timeoutReceivingStates.some((required) => required === state) && @@ -9092,6 +9282,13 @@ oracleProperty({ enters: [`bound-selector-prototype-${kind}` as const], build: (generated: Suite) => selectorPrototypeSuite(generated, kind), })), + // Law `constructed independence`, after an empty list and after a + // drawn one (maintainer decision, 2026-11-16). + ...([false, true] as const).map((listed) => ({ + name: `appended constructed history after ${listed ? `a drawn list` : `none`}`, + enters: [listed ? `appended-to-listed` : `appended-to-empty-list`], + build: (generated: Suite) => appendedPairSuite(generated, listed), + })), ], fixed: [ { From 2b3b4842298cbba047511012b51251f75f23a2be Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 13:28:18 +0000 Subject: [PATCH 2/9] fix(oracles): draw a campaign's generated histories apart from its constructed ones The generated unit ran fc.oneof over the arbitrary and each constructed history, weighted by the constructed count, so appending a constructed history moved every fixed seed's generated histories. The generated unit now draws from the arbitrary alone; each constructed history still runs in its own unit, built from the sample at seed plus its index. The self-oracle model's units rule drops the weighting (maintainer decision, 2026-11-16). Co-authored-by: Isaac --- .changeset/constructed-stream-independent.md | 7 ++++ packages/oracles/docs/oracle-checklist.md | 11 +++--- packages/oracles/src/oracle-property.ts | 29 +++++++-------- .../tests/oracle-property-self-oracle.test.ts | 35 +++++-------------- 4 files changed, 35 insertions(+), 47 deletions(-) create mode 100644 .changeset/constructed-stream-independent.md diff --git a/.changeset/constructed-stream-independent.md b/.changeset/constructed-stream-independent.md new file mode 100644 index 00000000..df12f26f --- /dev/null +++ b/.changeset/constructed-stream-independent.md @@ -0,0 +1,7 @@ +--- +'@tanstack/oracles': minor +--- + +A campaign's generated histories no longer depend on its constructed histories. The `generated` unit draws only from the property's arbitrary, so appending a constructed history, or changing one, keeps every fixed seed's generated histories, as the generator rules assume. Before, half of the generated unit's runs were constructed histories, weighted by how many there were, so appending one moved every run's draw. + +This shifts each fixed seed's `generated` unit once: a fixed seed now runs other generated histories than before, and a replay line printed before this change for a `generated` unit no longer replays. Each constructed history still runs in its own unit, built from the arbitrary's sample at the campaign's seed plus its index, unchanged. Constructed histories are no longer also explored over the `generated` unit's runs; a random seed builds each one from a new sample. diff --git a/packages/oracles/docs/oracle-checklist.md b/packages/oracles/docs/oracle-checklist.md index 31c11fd8..1e1df603 100644 --- a/packages/oracles/docs/oracle-checklist.md +++ b/packages/oracles/docs/oracle-checklist.md @@ -275,10 +275,13 @@ No requirement here can be met by declaring something out of reach. on, and a history MUST be constructed to enter each one. - **Framework:** Enforced. Every state in `states` must be named by a constructed history. Every campaign runs each constructed history, and one - that does not enter its named state fails. Half of the generated histories - are also constructed, so the histories after each state are explored. A - state that random histories reach only by chance goes in `measured`, which - is reported but not required. + that does not enter its named state fails. The generated histories are + drawn apart from the constructed ones, so at a fixed seed they do not + depend on the constructed list: appending a constructed history keeps + every fixed seed's generated histories. Each random seed builds the + constructed histories from new samples. A state that random histories + reach only by chance goes in `measured`, which is reported but not + required. - **Author:** Reach is a predicate over the plan or trace, or a production probe. A literal example is not reach. A `fixed` history is a regression, not reach. A law that needs a state entered at least N times in each diff --git a/packages/oracles/src/oracle-property.ts b/packages/oracles/src/oracle-property.ts index 010fb8a8..f5cd58b7 100644 --- a/packages/oracles/src/oracle-property.ts +++ b/packages/oracles/src/oracle-property.ts @@ -170,9 +170,15 @@ * constructed histories make reach required: a state that random histories * reach is measured, never gated. So every state in `states` must be named * by a constructed history, and a state the property only measures is - * declared in `measured` instead (ORC-015). Half of a campaign's generated - * histories are also constructed, so the histories after each required - * state are explored, not visited once. + * declared in `measured` instead (ORC-015). The constructed histories and + * the generated ones are independent streams: the `generated` unit draws + * only from the arbitrary, and constructed history k builds from the + * arbitrary's sample at the campaign's seed plus k. So at a fixed seed the + * generated histories do not depend on the constructed list, its length or + * its contents, and appending a constructed history changes no history that + * a fixed seed ran before (maintainer decision, 2026-11-16). A random seed + * builds each constructed history from a new sample, so the histories after + * a required state vary from run to run. * - A **fixed history** is a named history, such as a fixed regression. It may * name the states it `enters`, which its run must reach. Fixed and * constructed histories must be histories the arbitrary can take back @@ -2391,18 +2397,6 @@ function generatedTests( throw new Error( `${id}: no constructed history enters ${unconstructed.join(`, `)}. A required state needs a constructed history; a state that is only counted belongs in \`measured\`.`, ) - // Generated histories, half of them constructed. A constructed history - // shrinks through its generated input, as a mapped arbitrary does. - const arbitrary = - constructions.length === 0 - ? source.arbitrary - : fc.oneof( - { arbitrary: source.arbitrary, weight: constructions.length }, - ...constructions.map((construction) => ({ - arbitrary: source.arbitrary.map(construction.build), - weight: 1, - })), - ) for (const construction of constructions) { const undeclared = construction.enters.filter( (state) => declared !== undefined && !required.includes(state), @@ -2570,9 +2564,12 @@ function generatedTests( : timeout > 0 ? started + timeout / 2 : Infinity + // Every unit runs the property's own arbitrary: the `generated` unit + // draws from it alone, so the constructed list moves none of its + // histories, and a constructed unit's one history shrinks through it. const details = await fc.check( fc.asyncProperty( - arbitrary, + source.arbitrary, predicateFor(constructionOf, counts, shrinkUntil, rerun, violations), ), run, diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index 55599fda..866ba42d 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -33,8 +33,9 @@ * - `units` (maintainer decision, 2026-11-11, option D): a campaign's units run exactly the * histories that the campaign ran, each in its own test. The constructed unit of history k, for * seed S, runs one history: the construction built from the arbitrary's sample at seed S + k. - * The `generated` unit runs the stream that `fc.sample` draws from the property's arbitrary, with - * its constructed weighting, at seed S, for the property's `runs`. The model draws these with + * The `generated` unit runs the stream that `fc.sample` draws from the property's arbitrary alone, + * at seed S, for the property's `runs` (maintainer decision, 2026-11-16, which retired the + * constructed weighting of option D's design). The model draws these with * fast-check itself, never through the framework. A unit that fails ran that stream up to its * first failure, then its shrinks. The random units of a property run the one seed it logged: the * seed of the replay environment in a seeded run, and a drawn seed in a seedless run. @@ -1734,15 +1735,10 @@ const unitOf = (test: string): { seed?: number; name: string } | undefined => { } // The histories a unit draws at `seed`, in order: a constructed unit its one // construction, built from seed + its index; the `generated` unit the stream -// that fast-check samples from the property's arbitrary with its constructed -// weighting (each constructed history weighs 1, and the language weighs their -// count) for `runs` draws. Drawn with fast-check itself, never the framework. -// The weighting restates how production composes its arbitrary, which the -// design of option D authorizes the model to state (design-c-d.md, law -// `units`: "the property's arbitrary (with its constructed weighting)"). It -// is that composition, not a contract promise: the header promises only that -// half of a campaign's generated histories are constructed, and a change to -// the composition must change this model with it. +// that fast-check samples from the property's arbitrary alone, for `runs` +// draws. Drawn with fast-check itself, never the framework. The `generated` +// stream names no constructed history (law `constructed independence`, +// maintainer decision, 2026-11-16). function unitStream( spec: SyntheticSpec, seed: number, @@ -1754,22 +1750,7 @@ function unitStream( ) if (at >= 0) return [constructedAt(spec, spec.constructed[at]!.prefix, seed + at)] - const weighted = - spec.constructed.length === 0 - ? syntheticArbitrary - : fc.oneof( - { arbitrary: syntheticArbitrary, weight: spec.constructed.length }, - ...spec.constructed.map(({ prefix }) => ({ - arbitrary: syntheticArbitrary.map((items) => - [ - ...prefix, - ...items.map((item) => (spec.isolatedPlans ? item % 3 : item)), - ].slice(0, 3), - ), - weight: 1, - })), - ) - return fc.sample(weighted, { seed, numRuns: spec.source.runs }) + return fc.sample(syntheticArbitrary, { seed, numRuns: spec.source.runs }) } // The history each determinism unit runs twice: a generated source's fixed // histories, its constructed histories built from the first fixed seed, and From bdd7917e1bd44a85924b2a071c4bfe39b7ba8d24 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 13:58:13 +0000 Subject: [PATCH 3/9] test(oracles): keep the self-oracle's split-verdict and appended histories in the language The appended pair kept a drawn list of two, so the appended draft held three constructed histories, which the draft grammar cannot take back: keep the first. The split-verdict witnesses used seed 5, whose stream failed under law four only through the retired constructed weighting; derive the seed from the arbitrary instead. Co-authored-by: Isaac --- .../tests/oracle-property-self-oracle.test.ts | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index 866ba42d..d36c516e 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -8267,7 +8267,7 @@ const unitWitnessSuite = ( // Law `constructed independence`: a pair of drafts, the second the first with // one constructed history appended. The first draft's seed, runs, list, and // naming come from the generated suite, so random seeds vary them; a drawn -// list is kept to the ordinary items, so each of its histories enters the +// list keeps its first history only, and only the ordinary items, so each of its histories enters the // state of its first item. The appended history's prefix comes from the // suite's first fixed history, and it enters a state in some suites only, so // the pair's declared states differ in those. An enumerated first draft @@ -8281,7 +8281,9 @@ const appendedPairSuite = (generated: Suite, listed: boolean): Suite => { const tail = base.clock ? 1 : 2 const constructed = listed ? (base.constructed.length > 0 - ? base.constructed + ? // The draft grammar holds at most two constructed histories, and + // the appended draft adds one. + base.constructed.slice(0, 1) : [{ prefix: [tail], enters: [] }] ).map(({ prefix }) => { const items = ordinaryPrefix(prefix, tail) @@ -8312,6 +8314,23 @@ const appendedPairSuite = (generated: Suite, listed: boolean): Suite => { }, ]) } +// The first seed of the grammar's range (1 to 999) whose generated stream of +// `runs` histories first fails under law `four`: its first history that holds +// a 3, a 4, or a 6 holds a 4 first, as `verdictOf` judges a plain spec. The +// seed is derived from the arbitrary, not written down, so a change in how a +// campaign draws its `generated` stream moves the seed and keeps the state it +// reaches (law `constructed independence` moved every stream once). +const seedFailingUnderFour = (runs: number): number => { + const fails = (history: SyntheticHistory) => + history.find((entry) => entry === 3 || entry === 4 || entry === 6) + for (let seed = 1; seed <= 999; seed++) { + const failing = fc + .sample(syntheticArbitrary, { seed, numRuns: runs }) + .find((history) => fails(history) !== undefined) + if (failing && fails(failing) === 4) return seed + } + throw new Error(`no seed from 1 to 999 fails first under law four`) +} const mainUnitRequiredConstructed: Array<{ name: string enters: ReadonlyArray<(typeof mainUnitRequiredStates)[number]> @@ -8325,7 +8344,11 @@ const mainUnitRequiredConstructed: Array<{ return unitWitnessSuite( [ generatedDraft({ - source: { kind: `generated`, runs: 2, seed: 5 }, + source: { + kind: `generated`, + runs: 2, + seed: seedFailingUnderFour(2), + }, secondSeed: true, constructedNames: `prefixed`, constructed: [ @@ -10999,14 +11022,14 @@ oracleProperty({ [`timeout`], generatedDraft({ timing: `slow`, fixed: [at([0])] }), ), - // Units (maintainer decision, 2026-11-11, option D). At seed 5, the + // Units (maintainer decision, 2026-11-11, option D). At the seed, the // generated unit's first failing history holds a 4 first, and the // constructed unit's history starts with a 3. fixedHistory( `each unit decides its own first failure`, [`unit-split-verdict`, `campaign-violation`, `replay-probe`], generatedDraft({ - source: { kind: `generated`, runs: 2, seed: 5 }, + source: { kind: `generated`, runs: 2, seed: seedFailingUnderFour(2) }, constructed: [{ prefix: [3], enters: [] }], }), ), From d01a3464067ceffd9d176b489d3a9ad9587afb59 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 16:32:36 +0000 Subject: [PATCH 4/9] test(oracles): each constructed history runs its N bases from its own stream Extends law constructed independence and the units and campaign budget model rules (register M, 2026-11-16, "N bases per constructed history"): constructed history k at seed S runs the first N draws of the arbitrary's sample at S + k, N being the property's bases (default 5), and a bases that is not a whole number of at least 1 fails the load. The draft grammar draws bases and up to three constructed histories, and the appended pair keeps two before its append. RED on the one-base framework. Co-authored-by: Isaac --- .../oracle-property-self-oracle-fixture.ts | 7 + .../tests/oracle-property-self-oracle.test.ts | 203 ++++++++++++++---- 2 files changed, 174 insertions(+), 36 deletions(-) diff --git a/packages/oracles/tests/oracle-property-self-oracle-fixture.ts b/packages/oracles/tests/oracle-property-self-oracle-fixture.ts index 7bad497f..b38c217c 100644 --- a/packages/oracles/tests/oracle-property-self-oracle-fixture.ts +++ b/packages/oracles/tests/oracle-property-self-oracle-fixture.ts @@ -259,6 +259,12 @@ export type SyntheticSpec = { | { kind: `enumerated`; histories: ReadonlyArray } /** A second fixed seed, ten times the first, whose text the first's extends. */ secondSeed?: boolean + /** + * The bases each constructed history runs (the property's `bases`); absent + * leaves the framework's default. 0 and 1.5 are not whole numbers of at + * least 1, so they fail the load. + */ + bases?: number /** Required states; each must be entered by a constructed history. */ states: ReadonlyArray constructed: ReadonlyArray<{ @@ -1481,6 +1487,7 @@ export function syntheticProperty(spec: SyntheticSpec): void { .filter((state) => !spec.states.includes(state)) .map(declare), runs: spec.source.runs, + ...(spec.bases === undefined ? {} : { bases: spec.bases }), fixedSeeds: [ spec.source.seed, ...(spec.secondSeed ? [spec.source.seed * 10] : []), diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index d36c516e..eaf9c723 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -29,21 +29,27 @@ * wasted work. A generated source also fails to load when two of its tests would share a name: a * constructed history named `generated`, two constructed histories of one name, or two fixed * histories of one name (maintainer decision, 2026-11-11, option D). These come after its state - * checks and before law `membership`, in that order (analyst-chosen, 2026-11-11). + * checks and before law `membership`, in that order (analyst-chosen, 2026-11-11). A generated + * source whose `bases` is not a whole number of at least 1 fails to load, naming `bases`, after + * the pairing of `predict` with a probe-bound state and before its state checks (analyst-chosen, + * 2026-11-16). * - `units` (maintainer decision, 2026-11-11, option D): a campaign's units run exactly the * histories that the campaign ran, each in its own test. The constructed unit of history k, for - * seed S, runs one history: the construction built from the arbitrary's sample at seed S + k. + * seed S, runs N histories: the construction built from each of the first N draws of the + * arbitrary's sample at seed S + k, in order, where N is the property's `bases`, 5 by default + * (register M, 2026-11-16, "N bases per constructed history"). * The `generated` unit runs the stream that `fc.sample` draws from the property's arbitrary alone, - * at seed S, for the property's `runs` (maintainer decision, 2026-11-16, which retired the - * constructed weighting of option D's design). The model draws these with + * at seed S, for the property's `runs` (register M, 2026-11-16, "N bases per constructed + * history", which retired the constructed weighting of option D's design). The model draws these with * fast-check itself, never through the framework. A unit that fails ran that stream up to its * first failure, then its shrinks. The random units of a property run the one seed it logged: the * seed of the replay environment in a seeded run, and a drawn seed in a seedless run. - * - `constructed independence` (maintainer decision, 2026-11-16, review of #121, F2): at a fixed - * seed, the generated histories a campaign runs do not depend on its constructed list, its - * length or its contents, and each constructed history runs as declared, independently of the - * generated draw. So appending a constructed history keeps every fixed seed's histories, which - * the decisions register's generator rules assume. Judged on an appended pair: two properties + * - `constructed independence` (register M, 2026-11-16, "N bases per constructed history"; + * found by the review of #121, F2): at a fixed seed, the generated histories a campaign runs do + * not depend on its constructed list, its length or its contents, and each constructed history + * runs its N bases from a stream of its own, which neither the generated draws nor the rest of + * the list touch. So appending a constructed history keeps the fixed seed's generated histories + * and every other constructed history's bases. Judged on an appended pair: two properties * alike except that the second's constructed list is the first's with one history appended. At * each fixed seed, and at the seeded run's random seed, the `generated` unit and the unit of each * constructed history of the shorter list run the same histories in the same order in both, up @@ -149,9 +155,10 @@ * has its own determinism unit, which runs it twice and fails if its trace changes, * compared by their data encoding: `-0` is not `0`, and key order and * Map and Set order count, which a check that iterates them can see. - * - `campaign budget`: each passing constructed unit runs one example, and + * - `campaign budget`: each passing constructed unit runs one example per base (its property's + * `bases`, 5 by default), and * each passing generated unit runs its authored sample count. A wholly - * passing lane retains their sum: runs plus constructions, independently + * passing lane retains their sum: runs plus bases times constructions, independently * in ordinary/seedless and fixed/random lanes. Children discard parent replay * settings under both prefixes, so their multiplier is one. Path replay runs * one history; failures, shrinking, and timeouts owe no fixed call count. @@ -266,7 +273,10 @@ * unmapper, at the whole history or at each item; `.chain` and a filter are * not drawn, and a constructed history whose build leaves the language only * for some seeds is not drawn, so the run-time check of a later campaign's - * constructed histories has no witness here. The model learns which histories a test ran from the + * constructed histories has no witness here. A synthetic property holds at most three constructed + * histories, and an appended pair at most two before its append, so a design that changes only for + * longer lists has no witness; `bases` is drawn as the default, 1, 2, or 3, or as 0 or 1.5, which + * fail the load. The model learns which histories a test ran from the * synthetic driver's log, and relies on fast-check running a campaign's examples * (its constructed histories) first, in order. A replay or rerun probe covers * every failing campaign but only the first violating fixed or enumerated history of each @@ -495,6 +505,12 @@ type Draft = { enumeratedShape: `ok` | `extra key` | `probe state` nested: boolean strictModel: boolean + /** + * A generated source's `bases`: absent (the framework's default), a whole + * number of bases, or a value that is not a whole number of at least 1, + * which fails the load. + */ + bases: number | null } // `seedless`: one more child run, without `ORACLE_SEED`, records the routes // of each random campaign as an ordinary run draws its seed. @@ -656,9 +672,11 @@ const draft: fc.Arbitrary = fc.record({ weight: 1, }, ), + // Up to three, so a list long enough for a design that mixes only "when + // there are many" is drawn (review of #122, F5). constructed: fc.array( fc.record({ prefix: insideOrOutside(6), enters: fc.subarray(allStates) }), - { maxLength: 2 }, + { maxLength: 3 }, ), fixed: fc.array( fc.record( @@ -973,6 +991,13 @@ const draft: fc.Arbitrary = fc.record({ { weight: 12, arbitrary: fc.constant(false) }, { weight: 1, arbitrary: fc.constant(true) }, ), + // Most drafts keep the default number of bases; some declare one to three, + // and a few declare a value that fails the load. + bases: fc.oneof( + { weight: 6, arbitrary: fc.constant(null) }, + { weight: 3, arbitrary: fc.constantFrom(1, 2, 3) }, + { weight: 1, arbitrary: fc.constantFrom(0, 1.5) }, + ), }) const replayEnvironment: fc.Arbitrary = fc.record({ retired: fc.oneof( @@ -1192,6 +1217,7 @@ function resolveSuite({ id: syntheticIds[index]!, source: d.source, ...(generated && d.secondSeed ? { secondSeed: true } : {}), + ...(generated && d.bases !== null ? { bases: d.bases } : {}), states, constructed, fixed: generated @@ -1699,19 +1725,28 @@ const calibrationCandidates = ( ] } // Model law `units`. A synthetic constructed history is its prefix followed by -// a generated history, cut to three items; the generated history is the -// arbitrary's sample at the seed it is built from. -const constructedAt = ( +// a base, a generated history, cut to three items. +const builtFrom = ( spec: SyntheticSpec, prefix: SyntheticHistory, - seed: number, + base: SyntheticHistory, ) => [ ...prefix, - ...fc - .sample(syntheticArbitrary, { seed, numRuns: 1 })[0]! - .map((item) => (spec.isolatedPlans ? item % 3 : item)), + ...base.map((item) => (spec.isolatedPlans ? item % 3 : item)), ].slice(0, 3) +// A construction's first base: the arbitrary's first draw at `seed`, which +// registration, the determinism test, and the calibrations build from. +const constructedAt = ( + spec: SyntheticSpec, + prefix: SyntheticHistory, + seed: number, +) => + builtFrom( + spec, + prefix, + fc.sample(syntheticArbitrary, { seed, numRuns: 1 })[0]!, + ) // The seed every seeded child run gives the random units (`${prefix}_SEED`). const childSeed = 20251103 const fixedSeedsOf = (spec: SyntheticSpec): Array => @@ -1733,12 +1768,18 @@ const unitOf = (test: string): { seed?: number; name: string } | undefined => { const random = /^random seed: (.*)$/.exec(test) return random ? { name: random[1]! } : undefined } -// The histories a unit draws at `seed`, in order: a constructed unit its one -// construction, built from seed + its index; the `generated` unit the stream -// that fast-check samples from the property's arbitrary alone, for `runs` -// draws. Drawn with fast-check itself, never the framework. The `generated` -// stream names no constructed history (law `constructed independence`, -// maintainer decision, 2026-11-16). +// The number of bases each constructed history runs: the property's `bases`, +// or the framework's default of 5 (maintainer decision, register M, 2026-11-16, +// "N bases per constructed history"). +const defaultBases = 5 +const basesOf = (spec: SyntheticSpec) => spec.bases ?? defaultBases +// The histories a unit draws at `seed`, in order. Constructed history k's unit +// runs its construction over the first N draws of the arbitrary's sample at +// seed + k, N being its bases: a stream of its own, which neither the +// generated draws nor the rest of the list touch. The `generated` unit runs +// the stream that fast-check samples from the property's arbitrary alone, for +// `runs` draws. Drawn with fast-check itself, never the framework (law +// `units`, and law `constructed independence`). function unitStream( spec: SyntheticSpec, seed: number, @@ -1749,7 +1790,9 @@ function unitStream( (construction) => construction.name === name, ) if (at >= 0) - return [constructedAt(spec, spec.constructed[at]!.prefix, seed + at)] + return fc + .sample(syntheticArbitrary, { seed: seed + at, numRuns: basesOf(spec) }) + .map((base) => builtFrom(spec, spec.constructed[at]!.prefix, base)) return fc.sample(syntheticArbitrary, { seed, numRuns: spec.source.runs }) } // The history each determinism unit runs twice: a generated source's fixed @@ -2228,6 +2271,16 @@ function predictSpec( ) return { loads: false } if (generated) { + // A number of bases that is not a whole number of at least 1 would run a + // constructed history no times, or a part of one. + if ( + spec.bases !== undefined && + !(Number.isInteger(spec.bases) && spec.bases >= 1) + ) + return { + loads: false, + loadError: `${spec.id}: \`bases\` must be a whole number of at least 1`, + } const declared = new Set(spec.states) const entered = new Set( spec.constructed.flatMap((construction) => construction.enters), @@ -2851,12 +2904,20 @@ const timeoutReceivingKinds = [ const timeoutReceivingStates = timeoutReceivingKinds.map( (kind) => `timeout-input-${kind}` as const, ) -// Law `constructed independence`: the required states of an appended pair. -const appendedStates = [`appended-to-empty-list`, `appended-to-listed`] as const +// Law `constructed independence`: the required states of an appended pair, +// and of constructed units that run their bases (laws `units` and `campaign +// budget`): the default number, and a number the property declares. +const appendedStates = [ + `appended-to-empty-list`, + `appended-to-listed`, + `constructed-default-bases`, + `constructed-declared-bases`, +] as const const frameworkStates = [ ...timeoutReceivingStates, ...appendedStates, `appended-pair-whole-stream`, + `bases-load-error`, `timeout-presentation-equivalence`, `timeout-continuation-cut`, // A slow spec that fails to load, in a plan that presents timeouts: the @@ -3155,7 +3216,7 @@ function frameworkReach( ? spec.source.kind === `generated` ? spec.source.runs : 0 - : 1 + : basesOf(spec) if (test?.state !== `passed` || calls.length !== count) return false total += calls.length reached.add( @@ -3163,7 +3224,10 @@ function frameworkReach( ) return true }) - if (complete && total === spec.source.runs + spec.constructed.length) { + if ( + complete && + total === spec.source.runs + basesOf(spec) * spec.constructed.length + ) { reached.add(`passing-budget-${child.context.kind}-${lane}`) if (spec.constructed.length > 0) reached.add(`passing-budget-constructed`) @@ -3935,6 +3999,35 @@ function frameworkReach( ) reached.add(`appended-pair-whole-stream`) } + // A constructed unit that passed after running its property's bases, each + // base a history of its own; a property without `bases` runs the default. + for (const spec of plan.specs) { + const seen = run?.files[spec.id] + if (spec.source.kind !== `generated` || spec.slow) continue + if ( + !expected.get(spec.id)!.loads && + seen?.loadErrors.some((message) => message.includes(`\`bases\``)) + ) + reached.add(`bases-load-error`) + const [seed] = fixedSeedsOf(spec) + if (!seen?.loaded || seed === undefined || spec.constructed.length === 0) + continue + const ranBases = spec.constructed.every(({ name }) => { + const unit = fixedUnit(seed, name) + return ( + seen.tests.find((test) => test.name === unit)?.state === `passed` && + (run?.log ?? []).filter( + (entry) => entry.test === `${spec.id} > ${unit}`, + ).length === basesOf(spec) + ) + }) + if (ranBases) + reached.add( + spec.bases === undefined + ? `constructed-default-bases` + : `constructed-declared-bases`, + ) + } return reached } @@ -6364,7 +6457,7 @@ function childExpectation( ? 1 : unitOf(test.name)!.name === `generated` ? spec.source.runs - : 1 + : basesOf(spec) if (actualCalls !== owedCalls) return { failure: violation( @@ -7843,6 +7936,7 @@ const generatedDraft = (overrides: Partial): Draft => ({ labelStyle: `plain`, nested: false, strictModel: false, + bases: null, ...overrides, }) const fixedHistory = ( @@ -8267,7 +8361,7 @@ const unitWitnessSuite = ( // Law `constructed independence`: a pair of drafts, the second the first with // one constructed history appended. The first draft's seed, runs, list, and // naming come from the generated suite, so random seeds vary them; a drawn -// list keeps its first history only, and only the ordinary items, so each of its histories enters the +// list keeps its first two histories, and only the ordinary items, so each of its histories enters the // state of its first item. The appended history's prefix comes from the // suite's first fixed history, and it enters a state in some suites only, so // the pair's declared states differ in those. An enumerated first draft @@ -8281,9 +8375,9 @@ const appendedPairSuite = (generated: Suite, listed: boolean): Suite => { const tail = base.clock ? 1 : 2 const constructed = listed ? (base.constructed.length > 0 - ? // The draft grammar holds at most two constructed histories, and - // the appended draft adds one. - base.constructed.slice(0, 1) + ? // The draft grammar holds at most three constructed histories, + // and the appended draft adds one. + base.constructed.slice(0, 2) : [{ prefix: [tail], enters: [] }] ).map(({ prefix }) => { const items = ordinaryPrefix(prefix, tail) @@ -8331,6 +8425,34 @@ const seedFailingUnderFour = (runs: number): number => { } throw new Error(`no seed from 1 to 999 fails first under law four`) } +// A passing campaign whose constructed histories run their bases: ordinary +// items only (isolated plans keep the bases ordinary too), one or two +// constructed histories, and, when declared, the generated suite's own number +// of bases, or 1 to 3 from its seed when it declares none or an invalid one. +const basesSuite = (generated: Suite, declared: boolean): Suite => { + const base = generated.drafts[0]! + const tail = base.clock ? 1 : 2 + const drawn = + base.bases !== null && Number.isInteger(base.bases) && base.bases >= 1 + ? base.bases + : 1 + ((base.source.kind === `generated` ? base.source.seed : tail) % 3) + return unitWitnessSuite([ + generatedDraft({ + source: + base.source.kind === `generated` + ? base.source + : { kind: `generated`, runs: 2, seed: 30 + tail }, + isolatedPlans: true, + constructed: [ + { prefix: [0], enters: [`saw-0`] }, + ...(base.constructed.length > 1 + ? [{ prefix: [tail], enters: [] }] + : []), + ], + ...(declared ? { bases: drawn } : {}), + }), + ]) +} const mainUnitRequiredConstructed: Array<{ name: string enters: ReadonlyArray<(typeof mainUnitRequiredStates)[number]> @@ -9293,6 +9415,15 @@ oracleProperty({ enters: [listed ? `appended-to-listed` : `appended-to-empty-list`], build: (generated: Suite) => appendedPairSuite(generated, listed), })), + // Laws `units` and `campaign budget`: each constructed history runs its + // bases, the default or a declared number (register M, 2026-11-16). + ...([false, true] as const).map((declared) => ({ + name: `constructed histories run ${declared ? `declared` : `default`} bases`, + enters: [ + declared ? `constructed-declared-bases` : `constructed-default-bases`, + ], + build: (generated: Suite) => basesSuite(generated, declared), + })), ], fixed: [ { From 656442183fd6c7375f1be0658f4fe8606c8cecee Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 16:34:33 +0000 Subject: [PATCH 5/9] feat(oracles): each constructed history runs N bases from its own stream Constructed history k at seed S runs its construction over the first N draws of the arbitrary's sample at S + k, N being histories.bases (default 5; a value that is not a whole number of at least 1 fails the load). The stream is independent of the generated draws and of the rest of the list, so appending a constructed history keeps every other one's bases, and each constructed history is explored again (register M, 2026-11-16, "N bases per constructed history"). Also corrects the scheduler oracle's prose. Co-authored-by: Isaac --- .changeset/constructed-stream-independent.md | 4 +- packages/oracles/docs/oracle-checklist.md | 11 +-- packages/oracles/src/oracle-property.ts | 79 ++++++++++++++----- ...property-scheduler-oracle.property.test.ts | 7 +- 4 files changed, 70 insertions(+), 31 deletions(-) diff --git a/.changeset/constructed-stream-independent.md b/.changeset/constructed-stream-independent.md index df12f26f..51ec490a 100644 --- a/.changeset/constructed-stream-independent.md +++ b/.changeset/constructed-stream-independent.md @@ -2,6 +2,6 @@ '@tanstack/oracles': minor --- -A campaign's generated histories no longer depend on its constructed histories. The `generated` unit draws only from the property's arbitrary, so appending a constructed history, or changing one, keeps every fixed seed's generated histories, as the generator rules assume. Before, half of the generated unit's runs were constructed histories, weighted by how many there were, so appending one moved every run's draw. +A campaign's generated histories no longer depend on its constructed histories, and each constructed history runs several bases from a stream of its own. The `generated` unit draws only from the property's arbitrary. Constructed history k builds from each of the first `bases` draws of the arbitrary's sample at the campaign's seed plus k, and every base must enter the states it names. The new `histories.bases` option sets that number. It must be a whole number of at least 1 (default 5), or the property fails to load. So appending a constructed history keeps every fixed seed's generated histories and every other constructed history's bases. Before, half of the generated unit's runs were constructed histories, weighted by how many there were, so appending one moved every run's draw, and each constructed unit ran one history. -This shifts each fixed seed's `generated` unit once: a fixed seed now runs other generated histories than before, and a replay line printed before this change for a `generated` unit no longer replays. Each constructed history still runs in its own unit, built from the arbitrary's sample at the campaign's seed plus its index, unchanged. Constructed histories are no longer also explored over the `generated` unit's runs; a random seed builds each one from a new sample. +This shifts each fixed seed once, for properties that have at least one constructed history. Their `generated` unit now runs other histories, and a seed-and-path replay line printed before this change for that unit no longer replays the same history. A constructed unit keeps its first base, and its first replay path index still selects it. Replay lines for properties without constructed histories stay valid: their `generated` unit already drew from the arbitrary alone. Each constructed unit now runs `bases` histories, so a campaign takes longer; set `bases` lower for a property whose constructed histories are slow. diff --git a/packages/oracles/docs/oracle-checklist.md b/packages/oracles/docs/oracle-checklist.md index 1e1df603..dc6cb520 100644 --- a/packages/oracles/docs/oracle-checklist.md +++ b/packages/oracles/docs/oracle-checklist.md @@ -275,11 +275,12 @@ No requirement here can be met by declaring something out of reach. on, and a history MUST be constructed to enter each one. - **Framework:** Enforced. Every state in `states` must be named by a constructed history. Every campaign runs each constructed history, and one - that does not enter its named state fails. The generated histories are - drawn apart from the constructed ones, so at a fixed seed they do not - depend on the constructed list: appending a constructed history keeps - every fixed seed's generated histories. Each random seed builds the - constructed histories from new samples. A state that random histories + that does not enter its named state, on any of its bases, fails. Each + constructed history runs `bases` bases (5 by default) from a stream of its + own, and the generated histories are drawn apart from the constructed ones. + So at a fixed seed, appending a constructed history keeps the generated + histories and every other constructed history's bases, and the histories + after each required state are explored, not visited once. A state that random histories reach only by chance goes in `measured`, which is reported but not required. - **Author:** Reach is a predicate over the plan or trace, or a production diff --git a/packages/oracles/src/oracle-property.ts b/packages/oracles/src/oracle-property.ts index f5cd58b7..42d9bbc5 100644 --- a/packages/oracles/src/oracle-property.ts +++ b/packages/oracles/src/oracle-property.ts @@ -165,28 +165,32 @@ * driver must await the work it starts. * - A **constructed history** is built from a generated history so that it * reaches states that the laws need, and it names those declared states. - * Every campaign runs every constructed history once, in its own unit, and a - * constructed history that does not enter a named state fails. Only + * Every campaign runs every constructed history in its own unit, over a + * number of bases (`bases`, 5 by default), and a constructed history that + * does not enter a named state, on any base, fails. Only * constructed histories make reach required: a state that random histories * reach is measured, never gated. So every state in `states` must be named * by a constructed history, and a state the property only measures is * declared in `measured` instead (ORC-015). The constructed histories and * the generated ones are independent streams: the `generated` unit draws - * only from the arbitrary, and constructed history k builds from the - * arbitrary's sample at the campaign's seed plus k. So at a fixed seed the - * generated histories do not depend on the constructed list, its length or - * its contents, and appending a constructed history changes no history that - * a fixed seed ran before (maintainer decision, 2026-11-16). A random seed - * builds each constructed history from a new sample, so the histories after - * a required state vary from run to run. + * only from the arbitrary, and constructed history k builds from each of + * the first `bases` draws of the arbitrary's sample at the campaign's seed + * plus k. So at a fixed seed the generated histories do not depend on the + * constructed list, its length or its contents, appending a constructed + * history changes no other constructed history's bases, and each + * constructed history is explored over `bases` histories, not visited once + * (register M, 2026-11-16, "N bases per constructed history"). A history + * inserted mid-list moves the bases of the ones after it, so new + * constructed histories go at the end of the list. * - A **fixed history** is a named history, such as a fixed regression. It may * name the states it `enters`, which its run must reach. Fixed and * constructed histories must be histories the arbitrary can take back * (`canShrinkWithoutContext`), so they shrink like generated ones and cannot * cover a dimension that the grammar lacks; such a dimension belongs in the * grammar, or in the property's stated limits. The property checks this when - * it registers, for every fixed history and for the constructed histories - * built from the first fixed seed (or seed 0 when there is none), so a + * it registers, for every fixed history and for the first base of each + * constructed history at the first fixed seed (or seed 0 when there is + * none); a campaign checks every base it builds. So a * constructed history's `build` runs when the test file loads. It fails to * load naming the history, and a `build` that throws there fails to load * naming its constructed history @@ -604,6 +608,13 @@ export interface GeneratedHistories { /** One fixed-seed campaign per seed, beside the random campaign. */ fixedSeeds: ReadonlyArray constructed?: ReadonlyArray> + /** + * How many bases each constructed history runs in each campaign: the first + * `bases` draws of the arbitrary's sample at the campaign's seed plus the + * history's index, each built into one history. A whole number of at least + * 1; 5 when absent. + */ + bases?: number fixed?: ReadonlyArray> } @@ -1777,10 +1788,23 @@ function buildConstruction( seed: number, registering = false, ): THistory { - let history: THistory + return buildBases(id, arbitrary, construction, seed, 1, registering)[0]! +} + +/** A construction built from each of the first `count` draws at `seed`. */ +function buildBases( + id: string, + arbitrary: fc.Arbitrary, + construction: ConstructedHistory, + seed: number, + count: number, + registering = false, +): Array { + let histories: Array try { - const [generated] = fc.sample(arbitrary, { seed, numRuns: 1 }) - history = construction.build(generated!) + histories = fc + .sample(arbitrary, { seed, numRuns: count }) + .map((generated) => construction.build(generated)) } catch (error) { if (!registering) throw error throw new Error( @@ -1788,9 +1812,9 @@ function buildConstruction( { cause: error }, ) } - if (!arbitrary.canShrinkWithoutContext(history)) + if (histories.some((history) => !arbitrary.canShrinkWithoutContext(history))) throw untakeable(id, `constructed history "${construction.name}" built`) - return history + return histories } /** Stable across list insertion and reordering; no process-random replay route. */ @@ -2386,6 +2410,13 @@ function generatedTests( ) const options: BoundOptions = probes.size > 0 ? { ...unbound, probes } : unbound + // Each constructed history runs this many bases in every campaign (register + // M, 2026-11-16, "N bases per constructed history"). + const bases = source.bases ?? 5 + if (!(Number.isInteger(bases) && bases >= 1)) + throw new Error( + `${id}: \`bases\` must be a whole number of at least 1, the number of bases each constructed history runs in a campaign; it is ${reportText(bases)}.`, + ) const required = (source.states ?? []).map(stateName) const constructions = source.constructed ?? [] const declared = declaredOf(source) @@ -2494,6 +2525,11 @@ function generatedTests( registering, ) } + // A constructed unit's histories: its construction over the first `bases` + // draws of its own stream, the sample at seed + index, which neither the + // generated draws nor the other constructed histories touch. + const constructedBases = (seed: number, index: number) => + buildBases(id, source.arbitrary, constructions[index]!, seed + index, bases) // Registration checks every fixed history, then the constructed histories // built from the first fixed seed (or seed 0 when there is none), so a // history that cannot shrink fails the load rather than a full run. @@ -2508,8 +2544,8 @@ function generatedTests( const unitNames = [...constructedNames, `generated`] /** * Runs one unit of a campaign whose parameters are `base`. A constructed - * unit runs its one history, built from the campaign's seed, as an - * example with no generated runs; the `generated` unit runs the + * unit runs its bases, built from the campaign's seed, as examples with + * no generated runs; the `generated` unit runs the * campaign's generated runs, with no examples, so fast-check draws the * same histories that the whole campaign drew. A unit reports its reach * counts after it passes, and its rerun and replay commands when it fails. @@ -2534,12 +2570,13 @@ function generatedTests( const { reporter: _reporter, asyncReporter: _asyncReporter, ...rest } = base let run: typeof rest = rest if (index >= 0) { - const history = constructedAt(seed, index) + const histories = constructedBases(seed, index) // fast-check passes example inputs by reference, which identifies them. - constructionOf.set(history, constructions[index]!) + for (const history of histories) + constructionOf.set(history, constructions[index]!) run = withOracleFixedHistories( run.path === undefined ? { ...run, numRuns: 0 } : run, - [[history]], + histories.map((history) => [history]), ) } oracleRoute(oracleRouteOf(run)) diff --git a/packages/oracles/tests/oracle-property-scheduler-oracle.property.test.ts b/packages/oracles/tests/oracle-property-scheduler-oracle.property.test.ts index a52c38e9..2b1356c2 100644 --- a/packages/oracles/tests/oracle-property-scheduler-oracle.property.test.ts +++ b/packages/oracles/tests/oracle-property-scheduler-oracle.property.test.ts @@ -24,9 +24,10 @@ * task through `scheduleFrom`, waits until idle, and records the release * order and each task's outcome, or that `scheduleFrom` threw. The check * compares them. The declared state `a choice that names no queue` is entered - * by a constructed history that puts one first, so half of each campaign - * reaches it; the generated choices are mostly valid, so the release order is - * checked in most of the rest. + * by a constructed history that puts one first, which each campaign runs + * over its bases (5 by default); the generated choices are mostly valid, so + * they reach it only by chance, and the release order is checked in most of + * them. * * Limits: tasks are scheduled up front; tasks scheduled by released tasks' * continuations are not generated. From b91168ad3023a80be9c32cb290e55875a9e6f110 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 16:39:28 +0000 Subject: [PATCH 6/9] test(oracle-gaps): a constructed history enters authored agreement-only-kill The classify oracle measured the state, and its fixed seed's generated histories stopped reaching it once constructed histories left the generated unit. It is a G11 failure-gap class, so it is now required and entered by its own constructed history, appended at the end of the list. Co-authored-by: Isaac --- .../gap-report-classify-oracle.property.test.mjs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs index 5765025d..40343d47 100644 --- a/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-classify-oracle.property.test.mjs @@ -3230,6 +3230,7 @@ const required = [ `a survivor whose work fell`, `extra-work named past an unregistered rise`, `a rise exactly at the declared bound`, + `authored agreement-only-kill`, ] // One constructed history per outcome class: one mutant judged by one @@ -4000,6 +4001,18 @@ const constructed = [ baseline: `load`, }, ), + // An authored mutant that only the reach agreement rejects: a G11 + // failure-gap class, so a constructed history enters it (review of #122; + // the fixed seed's generated histories stopped reaching it once the + // constructed histories left the generated unit). + single( + `an authored mutant killed by reach agreement alone`, + [`authored agreement-only-kill`], + { + effect: { kind: `verdict`, law: `reach agreement` }, + authored: [{}], + }, + ), ] oracleProperty({ @@ -4016,7 +4029,6 @@ oracleProperty({ measured: [ ...[ `timeout`, - `agreement-only-kill`, `unregistered-kill`, `crashed`, `framework-error`, From 378b5e7435334bd6154d3e2e9f34d8c81c674dab Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 17:25:07 +0000 Subject: [PATCH 7/9] test(oracles): one base per constructed history in the two end-to-end oracles The gap report's end-to-end oracle and the framework self-oracle run real CLI and Vitest processes in each constructed history. At the default of five bases their fixed-seed shards took 18 to 30 and 27 minutes on an Arclet, past the 10-minute CI limit, so each declares bases: 1. Co-authored-by: Isaac --- .../oracles/oracle-gaps/gap-report-oracle.property.test.mjs | 5 +++++ packages/oracles/tests/oracle-property-self-oracle.test.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 0dcce7d9..d819e6bc 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -4576,6 +4576,11 @@ oracleProperty({ `a check that ran more than one batch`, ], runs: 2, + // One base per constructed history: each runs the CLI end to end, often + // for minutes, and five bases took a fixed-seed shard to 18 to 30 + // minutes on an Arclet, past the 10-minute CI limit (#122). Random seeds + // vary each one's base from run to run. + bases: 1, fixedSeeds: [20261105], constructed: withDeclaration([ { diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index eaf9c723..3c62f736 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -8666,6 +8666,11 @@ oracleProperty({ state !== `registration-warning-policy`, ), runs: 3, + // One base per constructed history: each spawns real Vitest runs, and + // five bases took the shard of campaign units to 27 minutes on an + // Arclet, past the 10-minute CI limit (#122). Random seeds vary each + // one's base from run to run. + bases: 1, fixedSeeds: [1], constructed: [ { From 774d915f5c95141cd2cac18b2403775ebdf12a4c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 17:55:22 +0000 Subject: [PATCH 8/9] test(oracle-gaps): three bases per constructed history in the static oracle Five bases took the static oracle's larger fixed-seed shard from 116 s to 530 s on an Arclet, close to the 10-minute CI limit. Co-authored-by: Isaac --- .../oracle-gaps/gap-report-static-oracle.property.test.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs index 5ccab440..72a4fa47 100644 --- a/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-static-oracle.property.test.mjs @@ -1450,6 +1450,10 @@ oracleProperty({ arbitrary: argumentHistories, states: required, runs: 40, + // Three bases per constructed history: each runs the CLI on a fixture + // package, and five bases took the larger fixed-seed shard from 116 s to + // 530 s on an Arclet, close to the 10-minute CI limit (#122). + bases: 3, fixedSeeds: [20261105], constructed: withDeclaration([ { From ba8ec15cc43bfd6a0426b9bd19784d1d644f8204 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 16 Nov 2026 17:58:10 +0000 Subject: [PATCH 9/9] docs(oracles): cite the register's N-bases entry as the self-oracle's authority Co-authored-by: Isaac --- packages/oracles/tests/oracle-property-self-oracle.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/oracles/tests/oracle-property-self-oracle.test.ts b/packages/oracles/tests/oracle-property-self-oracle.test.ts index 3c62f736..a9a88461 100644 --- a/packages/oracles/tests/oracle-property-self-oracle.test.ts +++ b/packages/oracles/tests/oracle-property-self-oracle.test.ts @@ -265,7 +265,9 @@ * the e2e speed plan, with Q2: one seed per property per process, Q3: a * `${prefix}_UNIT` variable that requires `${prefix}_PROPERTY`, and Q4: a path * replay of a generated source without `UNIT` fails at registration, and - * every replay line names its unit). + * every replay line names its unit); and the register's M entry of 2026-11-16, "N bases per + * constructed history" (v21.2 K18, with C-6 for appending at the end of the list), for laws + * `constructed independence`, `units`, and `campaign budget`. * * Limits: synthetic properties with histories of one to three integers, held * as an array, a bigint-tagged object, or a Map, one or two per run. An @@ -9414,7 +9416,7 @@ oracleProperty({ build: (generated: Suite) => selectorPrototypeSuite(generated, kind), })), // Law `constructed independence`, after an empty list and after a - // drawn one (maintainer decision, 2026-11-16). + // drawn one (register M, 2026-11-16, "N bases per constructed history"). ...([false, true] as const).map((listed) => ({ name: `appended constructed history after ${listed ? `a drawn list` : `none`}`, enters: [listed ? `appended-to-listed` : `appended-to-empty-list`],