diff --git a/.changeset/gap-cli-evidence-ran.md b/.changeset/gap-cli-evidence-ran.md new file mode 100644 index 0000000..14ac62c --- /dev/null +++ b/.changeset/gap-cli-evidence-ran.md @@ -0,0 +1,5 @@ +--- +'@tanstack/oracles': patch +--- + +The `oracle-gaps` report's `evidence` in `results.json` has a new count, `ran`: the units a check computed that ended without a cut. `cut` counts cut events (a cut copy that is confirmed alone is cut in its batch and again if its confirmation is cut), so `computed - cut` is not a count of runs. diff --git a/packages/oracles/README.md b/packages/oracles/README.md index d5fac0f..05a380b 100644 --- a/packages/oracles/README.md +++ b/packages/oracles/README.md @@ -29,4 +29,6 @@ Under the hook, each check also writes `check.pid` (the id of the process that r - For a settled or rejected batch: `after` (the sizes after its copies are retired), `copyNodes` (Vite module nodes that still name a copy, must be 0), and `nonCopyDup` (non-copy files with more than one module node in an environment, must be 0 beyond the measured plain-plus-`?raw` pair). - For a cut batch: `cut: true` and no `after`. Its context is replaced, not retired. +`ORACLE_GAPS_TEST_HOLD_OVERLAP` (a file path, unset in normal use) makes a daemon hold the first check it runs, before the check starts, until the daemon has accepted a second check request. Then it writes the number of check requests it has accepted to the path, and the check runs. It holds once per daemon, and it is released by that count, never by time. The two checks then overlap in the daemon, as two checks started at once do when the second request arrives in time. The hook changes no outcome: the second request waits in the queue, as it always does while a check runs. The e2e oracle sets it for its two checks at once in the edit sequence through the daemon, so that the overlap is held, not left to the timing of two processes (issue #82). + `ORACLE_GAPS_TEST_REJECT_BATCH` (a batch number from 1, unset in normal use) makes the gap CLI's n-th batch run reject, as a crash in Vitest would. It is a test-only fault: the batch's run completes, so its copies are in Vitest's and Vite's state, and then the run's promise rejects, which takes the same path a real crash takes. The run then ends as an internal error: exit 3, the done error line, and `error.log` in its output directory. The batch's copies are still retired and its retention line is still written. diff --git a/packages/oracles/oracle-gaps/check.mjs b/packages/oracles/oracle-gaps/check.mjs index a295b38..c9d8ee1 100644 --- a/packages/oracles/oracle-gaps/check.mjs +++ b/packages/oracles/oracle-gaps/check.mjs @@ -1619,13 +1619,16 @@ export default async (env) => { options.mutantLimit, ]), ) - // How the evidence was produced: units reused and computed, runs cut off - // at their time limit or for idleness (of them, `idle`), items rerun - // after a cut-off batch, and mutants that changed no value and so needed - // no run. + // How the evidence was produced: units reused and computed, of the + // computed ones those that ended without a cut (`ran`), runs cut off at + // their time limit or for idleness (of them, `idle`), items rerun after + // a cut-off batch, and mutants that changed no value and so needed no + // run. `cut` counts cut events: a cut copy confirmed alone is cut once + // in its batch and again if its confirmation is cut too. const evidence = { reused: 0, computed: 0, + ran: 0, cut: 0, idle: 0, spin: 0, @@ -1752,6 +1755,7 @@ export default async (env) => { }) evidence.computed += 1 if (result.timedOut) evidence.cut += 1 + else evidence.ran += 1 // A run cut off keeps its file's known inputs and any it named itself // (third review of #55, F-D). const paths = result.timedOut @@ -1818,6 +1822,7 @@ export default async (env) => { if (u.spin) evidence.spin += 1 remember(rest[k].id, u, batch.inputs[k]) evidence.computed += 1 + evidence.ran += 1 }) if (batch.timedOutAt === undefined) break // A copy past its first budget may only be slow: a loaded machine @@ -1907,6 +1912,7 @@ export default async (env) => { const u = confirmed(alone.units[0]) remember(hung.id, u, inputs) evidence.computed += 1 + evidence.ran += 1 return u } diff --git a/packages/oracles/oracle-gaps/daemon.mjs b/packages/oracles/oracle-gaps/daemon.mjs index 71837db..55c2f04 100644 --- a/packages/oracles/oracle-gaps/daemon.mjs +++ b/packages/oracles/oracle-gaps/daemon.mjs @@ -328,6 +328,19 @@ async function serve(pkg) { let parseArgs let idle let watch + // Test-only (ORACLE_GAPS_TEST_HOLD_OVERLAP, a path): the first check this + // daemon runs waits, before it starts, until the daemon has accepted a + // second check request, so two checks started at once overlap by a count, + // never by time. The release writes that count to the path. Once: a check + // after the first never waits. The hold changes no outcome; the second + // request waits in the queue, as it does whenever two checks overlap. + const overlapMark = process.env.ORACLE_GAPS_TEST_HOLD_OVERLAP + let accepted = 0 + let overlapHeld = false + let overlapped + const secondAccepted = new Promise((resolvePromise) => { + overlapped = resolvePromise + }) // Requests wait until the session is open. let opened let queue = new Promise((resolvePromise) => { @@ -382,6 +395,8 @@ async function serve(pkg) { queue = queue.then(() => shutdown(connection)) return } + accepted += 1 + if (accepted === 2) overlapped() queue = queue.then(async () => { clearTimeout(idle) // A request queued before the session failed hears why. @@ -410,6 +425,11 @@ async function serve(pkg) { const options = parseArgs(message.argv, message.cwd) if (options.pkg !== pkg) throw new Error(`this daemon serves ${pkg}, not ${options.pkg}`) + if (overlapMark !== undefined && !overlapHeld) { + overlapHeld = true + await secondAccepted + writeMark(overlapMark, `${accepted}`) + } stdout = await session.check(options, store, (line) => stderr.push(`${line}\n`), ) diff --git a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs index 7150836..a2a3f3e 100644 --- a/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs +++ b/packages/oracles/oracle-gaps/gap-report-model-oracle.mjs @@ -860,6 +860,13 @@ const drawn = fc.record({ // The batch whose run rejects: a constructed-only field (0 is no fault). Only the constructed history at // the end of the campaign sets it; the take-back below returns it to 0 (the generator rule, extended). rejectBatch: fc.constant(0), + // Whether two checks at once are held to overlap in the daemon (the + // test-only hold ORACLE_GAPS_TEST_HOLD_OVERLAP, issue #82): the first check + // a daemon serves waits until it has accepted a second request. A hook is + // never switched on by a drawn field, so the constant draws nothing and + // only the constructed edit sequence through the daemon sets it; the + // take-back below returns it to false. + holdOverlap: fc.constant(false), }) // The edit sequence over the witness module, the one list of steps that a @@ -932,6 +939,7 @@ const histories = drawn.map( pauseEngages: true, warmReplaced: false, rejectBatch: 0, + holdOverlap: false, } const earlier = history.steps.filter((step) => !laterEdits.includes(step)) return { @@ -939,6 +947,7 @@ const histories = drawn.map( pauseEngages: true, warmReplaced: false, rejectBatch: 0, + holdOverlap: false, steps: earlier.length > 0 ? earlier : null, laterSteps: later, } @@ -2969,6 +2978,8 @@ function resolveHistory(history) { ordinary: history.ordinary === true, // The batch whose run rejects (a test-only fault; 0 is none). rejectBatch: history.rejectBatch ?? 0, + // Whether two checks at once are held to overlap in the daemon. + holdOverlap: history.holdOverlap === true, // An `owns` key in an assertion's arguments, in the ordinary file: its // assertion form and whether its list is a literal, or none. matcherOwns: history.matcherOwns ?? `none`, diff --git a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs index 0dcce7d..4c25433 100644 --- a/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs +++ b/packages/oracles/oracle-gaps/gap-report-oracle.property.test.mjs @@ -266,7 +266,18 @@ * addon was added into an existing build/Release or a scoped one into * an existing scope directory each report * what the cold run reports; once `--stop-daemon` returns, no daemon - * process serves the package. + * process serves the package. The daemon runs one check at a time, so of + * two checks at once on one state, the one it runs second reuses every + * run the first stored and computes none that was not cut off (where no + * test reads the clock), by the CLI's own count of runs that ended + * without a cut. Which of the two it runs first is a race between two + * processes, outside the contract: the pair is recorded by what each + * check did, the check that computed first and ties by their records, + * never by which process started first (issue #82). The constructed edit + * sequence holds each pair in the daemon together, through a test-only + * hold released by the daemon's count of accepted requests, so its reach + * names a measured overlap, not two processes' timing. A completed check + * through the daemon names its own report in its done line. * 10. **The runner runs the package as Vitest does.** From any install path * (one with a space and `#` included), for any spelling of `--source` * (`./`, a redundant segment, an absolute path through a symbolic link @@ -1086,6 +1097,49 @@ function filesUnder(root, within = ``) { // of the repository), so a package, and a workspace dependency in it, keeps // nothing of a run. +/** + * How many runs a check computed that ended without a cut: the CLI's own + * count (`evidence.ran`). `computed - cut` counts cut events, not cut runs: + * a cut copy that is confirmed adds a cut and no computed run, so the + * difference can go below zero and hide a run computed again (review of + * #126, F3). + */ +const computedRuns = (run) => run.evidence?.ran ?? 0 + +/** + * What law 9 records of one check through the daemon: its exit, refusal, + * ending, report, and stream (law 15): both streams' lines, and the stdout's + * gap lines in sorted order (an order is not part of the contract), judged + * against its own results.json, and the first line of a failure, for the + * message (readRun has named the scratch package and CLI copy in it). + */ +const checkRecord = (run) => ({ + exit: run.exit, + refusal: run.refusal, + ending: run.ending, + report: run.report, + stream: run.stream, + stdoutStream: run.stdoutStream, + stdoutGaps: [...run.stdoutGaps].sort(), + stderr: run.stderr?.split(`\n`)[0], +}) + +/** + * Two checks at once, by role: the daemon's earlier check, which computed + * more runs, first. Checks that computed as many are ordered by their + * records, so the order is total and no tie falls back on which process the + * driver started first (review of #126, F6). + */ +const byRole = (a, b) => { + const work = computedRuns(b) - computedRuns(a) + if (work !== 0) return work + const [ra, rb] = [ + JSON.stringify(checkRecord(a)), + JSON.stringify(checkRecord(b)), + ] + return ra < rb ? -1 : ra > rb ? 1 : 0 +} + /** * Runs the CLI several times at once on one state of the fixture. `outs` * receives each call's output directory synchronously, before any call @@ -1158,6 +1212,23 @@ function namedScratch(text, names) { .replaceAll(/session-\d+/g, `session-`) } +// A run's own output directory, `out`, named `/` in each +// spelling the stack may use, wherever it is a whole path: not followed by +// a character that continues a directory's name (a word character, `-`, or +// a `.` before one). +function namedOut(text, out) { + if (out === undefined) return text + let named = text + for (const spelling of new Set([pathToFileURL(out).href, out, posix(out)])) { + const escaped = spelling.replaceAll(/[.*+?^${}()|[\]\\]/g, `\\$&`) + named = named.replaceAll( + new RegExp(`${escaped}(?![\\w-]|\\.\\w)`, `g`), + `/`, + ) + } + return named +} + /** What one run of the CLI reported, from its exit status and its files. */ // A gap line the CLI streams to stderr as its mutant is judged (stream.mjs): // its class, its line, and its id, after the stream's prefix (law 15). @@ -1230,6 +1301,11 @@ function readRun(dir, out, status, stderr, cli, stdout) { [dir, ``], ] const named = status === 0 ? undefined : namedScratch(`${stderr}`, scratch) + // The stream's names: the run's own output directory as `` too, and + // only as a whole path segment, so a sibling whose name it begins + // (`out-daemon-2-3` beside `out-daemon-2`) keeps its own name (review of + // #126, F4). + const namedStream = (text) => namedScratch(namedOut(text, out), scratch) let results let report let summary = `` @@ -1303,13 +1379,16 @@ function readRun(dir, out, status, stderr, cli, stdout) { line.startsWith(`oracle-gaps: config `), ), // The stream's grammar (law 15), recorded as relations so that a trace - // does not depend on the order in which contexts finished. - stream: streamOf(lines, (text) => namedScratch(text, scratch)), + // does not depend on the order in which contexts finished. The run's own + // output directory is named ``, a relation: two checks at once are + // given two directories, and which of them the daemon runs first is a + // race (issue #82), so the name the driver gave a check is no part of + // the trace, while a line that names another run's directory still + // shows it. + stream: streamOf(lines, namedStream), // The stdout's lines, judged as the stderr's are (review of #46, F1 and // F2): every line of either stream is a line of the stream. - stdoutStream: streamOf(`${stdout ?? ``}`.split(`\n`), (text) => - namedScratch(text, scratch), - ), + stdoutStream: streamOf(`${stdout ?? ``}`.split(`\n`), namedStream), // The output directory the run was given, as the trace names it: the // final line names its report there. outDir: namedScratch(out, scratch), @@ -1572,12 +1651,45 @@ async function incremental(dir, plan, cli, copy, observe, states) { // The daemon's checks run before the cold run: a daemon's state must not // depend on a cold run between its checks (a session fixed while // requests keep coming, `fixSession`). + // + // Two checks at once reach one daemon, which runs one check at a time, + // in the order their requests reach it. That order is a race between + // two processes and no part of the contract (daemon.mjs: "One check runs + // at a time"), so either check may be the one the daemon runs first. The + // pair is recorded by what each check did, not by which process the + // driver started first: the check that computed more runs comes first, + // as the daemon's earlier check, and the other is the later one. Each is + // still judged in full, and a test's console output (the `note test + // output` line) belongs to the check that ran the test (issue #82). + // + // A plan may hold the two in the daemon at once (`holdOverlap`, the + // test-only hold ORACLE_GAPS_TEST_HOLD_OVERLAP): the first check the + // daemon runs waits until the daemon has accepted the second request, a + // count, never a time (R57). The hold pins the overlap, not the order: + // either check may still be the one the daemon runs first. The daemon + // writes the count to the mark when it releases the hold. + const overlapMark = + twice && plan.holdOverlap + ? join(mkdtempSync(join(realpathSync(tmpdir()), `gap-overlap-`)), `n`) + : undefined + const hold = overlapMark + ? { env: { ORACLE_GAPS_TEST_HOLD_OVERLAP: overlapMark } } + : {} const [daemon, ...others] = twice - ? await runClisAtOnce(dir, files, [ - { ...how, daemon: true }, - { ...how, daemon: true, outName: `out-daemon-2` }, - ]) + ? ( + await runClisAtOnce(dir, files, [ + { ...how, daemon: true, ...hold }, + { ...how, daemon: true, outName: `out-daemon-2`, ...hold }, + ]) + ).sort(byRole) : await daemonChecks(step, files, how) + // Whether the daemon held its first check until it had accepted both. + const heldTogether = + overlapMark !== undefined && + existsSync(overlapMark) && + Number(readFileSync(overlapMark, `utf8`)) >= 2 + if (overlapMark) + rmSync(dirname(overlapMark), { recursive: true, force: true }) if (!cold.has(key)) cold.set(key, runCli(dir, files, how)) const coldRun = cold.get(key) // A check through the daemon of a state that nothing changed reuses @@ -1585,10 +1697,15 @@ async function incremental(dir, plan, cli, copy, observe, states) { // reused while its inputs are unchanged), so where no test reads the // clock it computes no run that was not cut off (third and fourth // reviews of #55, F-A and F-A2). The incremental check judges the count. + // The later of two checks at once is such a check: nothing changed + // after the daemon's earlier check of the same state, whose evidence is + // stored before it replies. So one of the two computes and the other + // reuses, whichever order the daemon took them in. + const reuseOwed = plan.tests.every((t) => !t.flaky) unchangedComputed = - step === `recheck` && plan.tests.every((t) => !t.flaky) - ? (daemon.evidence?.computed ?? 0) - (daemon.evidence?.cut ?? 0) - : undefined + step === `recheck` && reuseOwed ? computedRuns(daemon) : undefined + const laterComputed = + twice && reuseOwed ? computedRuns(others[0]) : undefined for (const [i, run] of [daemon, ...others].entries()) { // A daemon's call: the daemon the driver killed during it, if any, wrote // the lines of its own batches (daemonChecks). @@ -1612,7 +1729,7 @@ async function incremental(dir, plan, cli, copy, observe, states) { observe(`incremental`, { step: i === 0 ? step : `${step} (a second check at once)`, markerStored, - unchangedComputed: i === 0 ? unchangedComputed : undefined, + unchangedComputed: i === 0 ? unchangedComputed : laterComputed, midrun: i === 0 && step === `midrun` && midrunChecks ? { @@ -1647,25 +1764,26 @@ async function incremental(dir, plan, cli, copy, observe, states) { // between two runs of one history. stderr: coldRun.exit === 3 ? coldRun.stderr : undefined, }, - daemon: { - exit: run.exit, - refusal: run.refusal, - ending: run.ending, - report: run.report, - // The stream through the daemon (law 15): both streams' lines, - // and the stdout's gap lines, in sorted order (an order is not - // part of the contract), judged against its own results.json. - stream: run.stream, - stdoutStream: run.stdoutStream, - stdoutGaps: [...run.stdoutGaps].sort(), - // The first line of a failure, for the message; readRun has named - // the scratch package and CLI copy in it. - stderr: run.stderr?.split(`\n`)[0], - }, + daemon: checkRecord(run), }) } if (twice && step === `start`) states.add(`two checks started the daemon at once`) + // Two checks held in the daemon at once, the earlier of which computed + // runs and the later reused them: the state in which the order of the + // two matters, and in which a daemon that served an overlapping request + // from anything but the earlier check's store would compute again. The + // overlap is the hold's count, not a reading of two processes' timing. + if ( + twice && + heldTogether && + computedRuns(daemon) > 0 && + laterComputed === 0 && + (others[0].evidence?.reused ?? 0) > 0 + ) + states.add( + `of two checks held in the daemon at once, one computed and the other reused`, + ) if ( daemonConfigFaults.includes(plan.fault) && daemon.exit === 2 && @@ -2791,7 +2909,7 @@ function incrementalCheck(observations) { if (value.unchangedComputed > 0) return violation( `incremental`, - `the check through the daemon of a state that nothing changed computed ${value.unchangedComputed} runs that were not cut off, owed none`, + `the check through the daemon at ${step}, of a state that nothing changed, computed ${value.unchangedComputed} runs that were not cut off, owed none`, `unchanged reuse`, ) // A witness guard, not a law-9 requirement: the mid-run marker sits in @@ -2955,6 +3073,18 @@ function check(plan, observations, expected) { `greppable stream`, ends.extra, ) + // A completed check through the daemon names its own report in its + // done line: the summary in the output directory its CLI was given, + // which the trace names `` (readRun), never another check's. + if ( + daemon.exit === 0 && + !daemon.stream.last.endsWith(` report //summary.md`) + ) + return violation( + `incremental`, + `through the daemon at ${value.step}, the done line ${JSON.stringify(daemon.stream.last)} does not name the check's own report, //summary.md`, + `greppable stream`, + ) // The check that a mid-run edit overlapped streams as any check does. const duringEnds = value.midrun && endsBreach(value.midrun.during) if (duringEnds) @@ -4168,6 +4298,7 @@ const incrementalStates = [ `a check after the OS cleaned the daemon's state`, `two checks started the daemon at once`, `the daemon ran from a path with a space and #`, + `of two checks held in the daemon at once, one computed and the other reused`, ] // Law 9's states that a constructed history of their own reaches, not the // edit sequence: the mid-run edit, and an edit to data a test reads through @@ -4767,8 +4898,11 @@ oracleProperty({ fault: `none`, stability: `none`, // Two checks at once start the daemon, from a CLI installed under - // a path with a space and `#`. + // a path with a space and `#`. Each pair of checks at once is held + // in the daemon together (`holdOverlap`), so the later request + // arrives while the earlier check is in flight, by a count. concurrentStart: true, + holdOverlap: true, cliPath: `spaced`, native: `none`, laterSteps: NO_LATER_STEPS, diff --git a/packages/oracles/oracle-gaps/history-fingerprint.json b/packages/oracles/oracle-gaps/history-fingerprint.json index 20fef65..b6c1e39 100644 --- a/packages/oracles/oracle-gaps/history-fingerprint.json +++ b/packages/oracles/oracle-gaps/history-fingerprint.json @@ -57,7 +57,8 @@ ["warmReplaced", "ed6866229ecc4969"], ["testRoot", "e18db063421a75f1"], ["loadPlugin", "ed6866229ecc4969"], - ["rejectBatch", "e5386ea9402dd268"] + ["rejectBatch", "e5386ea9402dd268"], + ["holdOverlap", "ed6866229ecc4969"] ], "histories": [ ["functions", "66c0e9c56951c377"], @@ -117,6 +118,7 @@ ["warmReplaced", "ed6866229ecc4969"], ["testRoot", "e18db063421a75f1"], ["loadPlugin", "ed6866229ecc4969"], - ["rejectBatch", "e5386ea9402dd268"] + ["rejectBatch", "e5386ea9402dd268"], + ["holdOverlap", "ed6866229ecc4969"] ] }