From 53a61f5ade27cbd57cddae3acb225d53a81bcb1f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:02:49 +0000 Subject: [PATCH 1/2] Keep a tail-only test report's failing tests in the artifact summary The artifact summary keeps a test report's last 64 KiB. The comparator report is about 450 KiB, so #826's one failing test (reported early) was missing from its retained summary, and the failure needed a local reproduction to find. When a test summary is cut to its tail, the summary also writes .failures.txt: every TAP `not ok` line with its diagnostic block, within the same per-file and total limits, recorded with derived_from and the failure count. A whole report or a passing one gets no excerpt. Co-Authored-By: Claude Opus 5.5 --- .../server-parity/artifact-summary.mjs | 40 ++++++++++++++++++- .../Unit/ServerParityArtifactSummaryTest.mjs | 25 ++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/scripts/conformance/server-parity/artifact-summary.mjs b/scripts/conformance/server-parity/artifact-summary.mjs index a2a24cac3..d0038fbd3 100644 --- a/scripts/conformance/server-parity/artifact-summary.mjs +++ b/scripts/conformance/server-parity/artifact-summary.mjs @@ -1,4 +1,4 @@ -import {constants, existsSync, lstatSync, mkdirSync, openSync, readSync, readdirSync, closeSync, writeFileSync} from 'node:fs'; +import {constants, existsSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, closeSync, writeFileSync} from 'node:fs'; import {basename, dirname, join, relative, resolve} from 'node:path'; import {pathToFileURL} from 'node:url'; @@ -10,7 +10,7 @@ function priority(path, failed) { const name = basename(path); if (/\.status$/.test(name) || /-status\.txt$/.test(name)) return -1; if (failed && /(?:^|\/)query-authority\/clients\/(?:php|rust)\/[^/]+\/(?:client\.stderr|observer\.log)$/.test(path)) return 0; - if (/^(rust|mysql|postgres|comparator|codec-load|artifact-summary)(-execution)?-tests\.txt$/.test(name)) return 0; + if (TEST_SUMMARY.test(name)) return 0; if (/^(comparison|compare)([-.].*)?\.json$/.test(name) || /-comparison\.json$/.test(name)) return 1; if (failed && (/^(?:php|embedded|rust)-record\.txt$/.test(name) || /\.failure-excerpt\.txt$/.test(name) || /^sdk-(?:signals|queries)-profile\.log$/.test(name))) return 1; @@ -22,6 +22,28 @@ function priority(path, failed) { return null; } +const TEST_SUMMARY = /^(rust|mysql|postgres|comparator|codec-load|artifact-summary)(-execution)?-tests\.txt$/; + +// A node:test (TAP) report's failing tests: each `not ok` line with its +// indented YAML diagnostic block, ending at the block's `...` line. A large +// report keeps only its tail, which can miss a failure reported early. +export function tapFailures(text) { + const lines = text.split('\n'); + const blocks = []; + for (let index = 0; index < lines.length; index++) { + const match = /^(\s*)not ok \d+ /.exec(lines[index]); + if (!match) continue; + const block = [lines[index]]; + const end = `${match[1]} ...`; + for (let next = index + 1; next < lines.length && next <= index + 200; next++) { + block.push(lines[next]); + if (lines[next] === end) break; + } + blocks.push(block.join('\n')); + } + return blocks; +} + function inventory(root, path = root, files = []) { if (!lstatSync(path).isDirectory()) throw new Error('Evidence root must be a directory.'); for (const entry of readdirSync(path, {withFileTypes: true})) { @@ -70,6 +92,20 @@ export function summarizeEvidence(source, destination, {outcome = 'unknown', rev } finally { closeSync(descriptor); } retained.push({path: file.path, source_bytes: file.source_bytes, retained_bytes: size, tail_only: size < file.source_bytes}); bytes += size; + // A tail-only test report also keeps its failing tests, bounded. + if (size < file.source_bytes && TEST_SUMMARY.test(basename(file.path)) && retained.length < ENTRY_LIMIT) { + const failures = tapFailures(readFileSync(join(source, file.path), 'utf8')); + if (failures.length > 0) { + const excerpt = Buffer.from(`${failures.join('\n')}\n`).subarray(0, Math.min(FILE_LIMIT, CONTENT_LIMIT - bytes)); + if (excerpt.length > 0) { + const path = `${file.path}.failures.txt`; + writeFileSync(join(destination, path), excerpt); + retained.push({path, source_bytes: file.source_bytes, retained_bytes: excerpt.length, tail_only: false, + derived_from: file.path, failures: failures.length}); + bytes += excerpt.length; + } + } + } } const summary = { schema: 'durable-workflow.server-parity-artifact-summary.v1', outcome, revision, diff --git a/tests/Unit/ServerParityArtifactSummaryTest.mjs b/tests/Unit/ServerParityArtifactSummaryTest.mjs index 7ec275f81..971366bc0 100644 --- a/tests/Unit/ServerParityArtifactSummaryTest.mjs +++ b/tests/Unit/ServerParityArtifactSummaryTest.mjs @@ -151,3 +151,28 @@ test('a failed record outline survives a full collection of setup logs and stays summarizeEvidence(source, success, {outcome: 'success'}); assert.equal(existsSync(join(success, 'legacy-marker/php/record.failure-excerpt.txt')), false); }); + +test('a tail-only test report also keeps its failing tests', t => { + const {source, output} = fixture(t); + const failure = [ + 'not ok 59 - only the adapter job caches', + ' ---', + " location: 'tests/Unit/ServerParityAdapterCacheTest.mjs:22:1'", + " error: 'Expected values to be strictly deep-equal'", + ' ...', + ].join('\n'); + const passing = Array.from({length: 4000}, (_, index) => `# Subtest: passing ${index}\nok ${index + 100} - passing ${index}\n ---\n duration_ms: 0.1\n ...`).join('\n'); + writeFileSync(join(source, 'comparator-tests.txt'), `${failure}\n${passing}\n# pass 4000\n# fail 1\n`); + writeFileSync(join(source, 'rust-tests.txt'), `${'ok 1 - fine\n'.repeat(10000)}# fail 0\n`); + const small = 'not ok 1 - small failure\n ---\n ...\n# fail 1\n'; + writeFileSync(join(source, 'codec-load-tests.txt'), small); + const summary = summarizeEvidence(source, output, {outcome: 'failure'}); + const tail = readFileSync(join(output, 'comparator-tests.txt'), 'utf8'); + assert.equal(tail.includes('not ok 59'), false, 'the failure is outside the retained tail'); + assert.equal(readFileSync(join(output, 'comparator-tests.txt.failures.txt'), 'utf8'), `${failure}\n`); + const derived = summary.retained.find(file => file.path === 'comparator-tests.txt.failures.txt'); + assert.deepEqual([derived.derived_from, derived.failures, derived.tail_only], ['comparator-tests.txt', 1, false]); + assert.equal(existsSync(join(output, 'rust-tests.txt.failures.txt')), false, 'a passing report has no failures file'); + assert.equal(readFileSync(join(output, 'codec-load-tests.txt'), 'utf8'), small); + assert.equal(existsSync(join(output, 'codec-load-tests.txt.failures.txt')), false, 'a whole report needs no excerpt'); +}); From b2b6e7531919c6512163d6a06ba00ff8318c5e63 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:43:14 +0000 Subject: [PATCH 2/2] Stream the failure scan and keep derived excerpts out of omission counts Cloud's review of #833 (comment 6461500906) found two problems: - the failure scan read the whole report with readFileSync and split it, so a large report defeated the summary's memory bounds; - a derived excerpt counted as a retained source file, so one failing report gave scanned_files 1, retained 2, omitted_files -1. scanTapFailures now reads the report in 64 KiB chunks through a UTF-8 StringDecoder, one line of at most 8 KiB at a time. It keeps at most the excerpt limit and 200 lines per block, and still counts failures past the limit. omitted_files counts source files only. New test: a 256 MiB sparse report with an early failure gives the exact excerpt and scanned 1 / retained 2 / omitted 0. A 1 GiB sparse report scans at 77 MiB peak RSS. Artifact summary tests pass 9/9. Co-Authored-By: Claude Opus 5.5 --- .../server-parity/artifact-summary.mjs | 77 ++++++++++++++----- .../Unit/ServerParityArtifactSummaryTest.mjs | 16 +++- 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/scripts/conformance/server-parity/artifact-summary.mjs b/scripts/conformance/server-parity/artifact-summary.mjs index d0038fbd3..424392a1b 100644 --- a/scripts/conformance/server-parity/artifact-summary.mjs +++ b/scripts/conformance/server-parity/artifact-summary.mjs @@ -1,6 +1,7 @@ -import {constants, existsSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, closeSync, writeFileSync} from 'node:fs'; +import {constants, existsSync, lstatSync, mkdirSync, openSync, readSync, readdirSync, closeSync, writeFileSync} from 'node:fs'; import {basename, dirname, join, relative, resolve} from 'node:path'; import {pathToFileURL} from 'node:url'; +import {StringDecoder} from 'node:string_decoder'; const FILE_LIMIT = 64 * 1024; const CONTENT_LIMIT = 2 * 1024 * 1024; @@ -26,22 +27,56 @@ const TEST_SUMMARY = /^(rust|mysql|postgres|comparator|codec-load|artifact-summa // A node:test (TAP) report's failing tests: each `not ok` line with its // indented YAML diagnostic block, ending at the block's `...` line. A large -// report keeps only its tail, which can miss a failure reported early. -export function tapFailures(text) { - const lines = text.split('\n'); - const blocks = []; - for (let index = 0; index < lines.length; index++) { - const match = /^(\s*)not ok \d+ /.exec(lines[index]); - if (!match) continue; - const block = [lines[index]]; - const end = `${match[1]} ...`; - for (let next = index + 1; next < lines.length && next <= index + 200; next++) { - block.push(lines[next]); - if (lines[next] === end) break; +// report keeps only its tail, which can miss a failure reported early. The +// report is read in fixed chunks, one bounded line at a time: memory stays +// within `limit` plus one chunk and one line, whatever the report's size. +// Failures past the excerpt limit are still counted. +const SCAN_CHUNK = 64 * 1024; +const LINE_LIMIT = 8 * 1024; +const BLOCK_LINES = 200; + +export function scanTapFailures(path, limit) { + const descriptor = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const chunk = Buffer.alloc(SCAN_CHUNK); + const decoder = new StringDecoder('utf8'); + const kept = []; + let keptBytes = 0; + let count = 0; + let pending = ''; + let block = null; + const keep = text => { + const size = Buffer.byteLength(text) + 1; + if (keptBytes + size > limit) return; + kept.push(text); + keptBytes += size; + }; + const line = text => { + if (block !== null) { + if (block.lines < BLOCK_LINES) { + keep(text); + block.lines++; + } + if (text === block.end || block.lines >= BLOCK_LINES) block = null; + return; } - blocks.push(block.join('\n')); - } - return blocks; + const match = /^(\s*)not ok \d+ /.exec(text); + if (match) { + count++; + keep(text); + block = {end: `${match[1]} ...`, lines: 1}; + } + }; + try { + let read; + while ((read = readSync(descriptor, chunk, 0, SCAN_CHUNK, null)) > 0) { + const parts = (pending + decoder.write(chunk.subarray(0, read))).split('\n'); + pending = parts.pop().slice(0, LINE_LIMIT); + for (const part of parts) line(part.slice(0, LINE_LIMIT)); + } + pending = (pending + decoder.end()).slice(0, LINE_LIMIT); + if (pending !== '') line(pending); + } finally { closeSync(descriptor); } + return {text: kept.length > 0 ? `${kept.join('\n')}\n` : '', count}; } function inventory(root, path = root, files = []) { @@ -94,14 +129,14 @@ export function summarizeEvidence(source, destination, {outcome = 'unknown', rev bytes += size; // A tail-only test report also keeps its failing tests, bounded. if (size < file.source_bytes && TEST_SUMMARY.test(basename(file.path)) && retained.length < ENTRY_LIMIT) { - const failures = tapFailures(readFileSync(join(source, file.path), 'utf8')); - if (failures.length > 0) { - const excerpt = Buffer.from(`${failures.join('\n')}\n`).subarray(0, Math.min(FILE_LIMIT, CONTENT_LIMIT - bytes)); + const failures = scanTapFailures(join(source, file.path), Math.min(FILE_LIMIT, CONTENT_LIMIT - bytes)); + if (failures.count > 0) { + const excerpt = Buffer.from(failures.text); if (excerpt.length > 0) { const path = `${file.path}.failures.txt`; writeFileSync(join(destination, path), excerpt); retained.push({path, source_bytes: file.source_bytes, retained_bytes: excerpt.length, tail_only: false, - derived_from: file.path, failures: failures.length}); + derived_from: file.path, failures: failures.count}); bytes += excerpt.length; } } @@ -111,7 +146,7 @@ export function summarizeEvidence(source, destination, {outcome = 'unknown', rev schema: 'durable-workflow.server-parity-artifact-summary.v1', outcome, revision, scanned_files: files.length, scanned_bytes: files.reduce((sum, file) => sum + file.source_bytes, 0), retained_bytes: bytes, retained, - omitted_files: files.length - retained.length, + omitted_files: files.length - retained.filter(file => file.derived_from === undefined).length, limits: {file_bytes: FILE_LIMIT, content_bytes: CONTENT_LIMIT, entries: ENTRY_LIMIT, excluded_directories: ['vendor', 'node_modules', 'target', '.git', 'objects']}, scope: 'Test summaries and bounded diagnostic tails only. Raw records, HTTP journals, databases and payload bytes are not retained. This summary does not rerun or qualify assertions.', diff --git a/tests/Unit/ServerParityArtifactSummaryTest.mjs b/tests/Unit/ServerParityArtifactSummaryTest.mjs index 971366bc0..dce60536a 100644 --- a/tests/Unit/ServerParityArtifactSummaryTest.mjs +++ b/tests/Unit/ServerParityArtifactSummaryTest.mjs @@ -1,6 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync} from 'node:fs'; +import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, truncateSync, writeFileSync} from 'node:fs'; import {join} from 'node:path'; import {tmpdir} from 'node:os'; import {summarizeEvidence} from '../../scripts/conformance/server-parity/artifact-summary.mjs'; @@ -176,3 +176,17 @@ test('a tail-only test report also keeps its failing tests', t => { assert.equal(readFileSync(join(output, 'codec-load-tests.txt'), 'utf8'), small); assert.equal(existsSync(join(output, 'codec-load-tests.txt.failures.txt')), false, 'a whole report needs no excerpt'); }); + +test('a failure excerpt is derived output and is read in bounded chunks', t => { + const {source, output} = fixture(t); + const failure = 'not ok 7 - early λ failure\n ---\n error: boom\n ...\n'; + const report = join(source, 'comparator-tests.txt'); + writeFileSync(report, failure); + // A sparse 256 MiB report: the scan must stream it, not load it whole. + truncateSync(report, 256 * 1024 * 1024); + const summary = summarizeEvidence(source, output, {outcome: 'failure'}); + assert.equal(readFileSync(join(output, 'comparator-tests.txt.failures.txt'), 'utf8'), failure); + assert.deepEqual([summary.scanned_files, summary.retained.length, summary.omitted_files], [1, 2, 0], + 'derived output does not count against source omission'); + assert.equal(summary.retained.find(file => file.derived_from).failures, 1); +});