From 288d8d8f50cac3e404f47e92f0a3147a6c0e8efa Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 15:58:12 +0200 Subject: [PATCH 1/2] test: schedule WPT variants individually Discover WPT tasks through their existing JavaScript drivers so the Python runner can schedule, report, and rerun generated paths. Assisted-by: Codex Signed-off-by: Filip Skokan --- test/common/wpt.js | 80 +++++++++++--- test/parallel/test-common-wpt-runner.js | 133 ++++++++++++++++++++++++ test/tools/test_wpt_runner.py | 129 +++++++++++++++++++++++ test/wpt/README.md | 9 ++ test/wpt/test-compression.js | 3 +- test/wpt/test-streams.js | 3 +- test/wpt/test-wasm-jsapi.mjs | 3 +- test/wpt/test-webcrypto.js | 3 +- test/wpt/test-webstorage.js | 2 +- test/wpt/testcfg.py | 103 +++++++++++++++++- tools/test.py | 15 ++- 11 files changed, 455 insertions(+), 28 deletions(-) create mode 100644 test/parallel/test-common-wpt-runner.js create mode 100644 test/tools/test_wpt_runner.py diff --git a/test/common/wpt.js b/test/common/wpt.js index 4ad788ae029d..69c93b1cbd2e 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -96,8 +96,8 @@ class ReportResult { // Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation // for more details. class WPTReport { - constructor(testPath) { - this.filename = `report-${testPath.replaceAll('/', '-')}.json`; + constructor(testPath, suffix = '') { + this.filename = `report-${testPath.replaceAll('/', '-')}${suffix}.json`; this.filepath = path.join(__dirname, `../../out/wpt/${this.filename}`); /** @type {Map} */ this.results = new Map(); @@ -608,7 +608,7 @@ class StatusLoader { return result; } - load() { + load(source) { const dir = path.join(__dirname, '..', 'wpt'); let result; @@ -625,7 +625,7 @@ class StatusLoader { this.rules.addRules(result); const subDir = fixtures.path('wpt', this.path); - const list = this.grep(subDir); + const list = source === undefined ? this.grep(subDir) : [path.join(subDir, source)]; for (const file of list) { const relativePath = path.relative(subDir, file); const match = this.rules.match(relativePath); @@ -813,14 +813,29 @@ const backends = { }; class WPTRunner { - constructor(path, { - concurrency = os.availableParallelism() - 1 || 1, - backend = 'thread', - } = {}) { + constructor(path, options = {}) { + let { + concurrency = os.availableParallelism() - 1 || 1, + backend = 'thread', + } = options; if (!Number.isInteger(concurrency) || concurrency < 1) { throw new TypeError('WPT concurrency must be a positive integer'); } + if (process.env.NODE_TEST_WPT !== undefined) { + this.managed = JSON.parse(process.env.NODE_TEST_WPT); + if (!this.managed || !['list', 'run'].includes(this.managed.mode) || + (this.managed.mode === 'run' && + (['source', 'key'].some((key) => + typeof this.managed[key] !== 'string' || !this.managed[key]) || + (this.managed.variant !== undefined && typeof this.managed.variant !== 'string')))) { + throw new Error('Invalid NODE_TEST_WPT configuration'); + } + } + this.isListing = this.managed?.mode === 'list'; + this.serial = options.concurrency === 1; + if (this.managed?.mode === 'run') concurrency = 1; + // RISC-V has very limited virtual address space in the currently common // sv39 mode, in which we can only create a very limited number of wasm // memories(27 from a fresh node repl). Limit the concurrency to avoid @@ -860,7 +875,7 @@ class WPTRunner { this.initScript = null; this.status = new StatusLoader(path); - this.status.load(); + this.status.load(this.managed?.mode === 'run' ? this.managed.source : undefined); this.statusFile = this.status.statusFile; this.specs = new Set(this.status.specs); @@ -872,8 +887,9 @@ class WPTRunner { this.subtestCounts = { passed: 0, failed: 0, expectedFailures: 0, skipped: 0, unexpectedPasses: 0 }; - if (process.env.WPT_REPORT != null) { - this.report = new WPTReport(path); + if (process.env.WPT_REPORT != null && !this.isListing) { + const suffix = this.managed ? `-${process.env.TEST_SERIAL_ID || process.pid}` : ''; + this.report = new WPTReport(path, suffix); } } @@ -958,7 +974,39 @@ class WPTRunner { // TODO(joyeecheung): work with the upstream to port more tests in .html // to .js. async runJsTests() { + if (this.isListing) { + const groups = new Map(); + for (const spec of this.specs) { + const key = spec.getStatusKey(); + if (!groups.has(key)) groups.set(key, []); + groups.get(key).push(spec); + } + const tests = [...groups.values()].flatMap((specs) => { + // Strict expected failures are checked across all query variants. + // Keep that group together; all other variants are independent tasks. + const grouped = specs.some((spec) => + spec.failedTests.some((name) => isUnexpectedPass(spec, name))); + return (grouped ? [specs[0]] : specs).map((spec) => { + const id = spec.getTestPath().slice(this.path.length + 1); + return { + source: spec.filename.split(path.sep).join('/'), + key: spec.getStatusKey(), + ...(grouped ? {} : { variant: spec.variant }), + id: grouped ? id.split('?')[0] : id, + }; + }); + }); + console.log(`NODE_TEST_WPT_MANIFEST:${JSON.stringify({ + version: 1, + serial: this.serial, + tests: tests.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)), + })}`); + return; + } const queue = this.buildQueue(); + if (this.managed && queue.length === 0) { + console.log('1..0 # SKIP No runnable WPT variants'); + } const run = limit(this.concurrency); const jobs = []; @@ -1313,11 +1361,19 @@ class WPTRunner { buildQueue() { const queue = []; this.skippedSpecCount = 0; - const arg = process.argv[2]; + const key = this.managed?.mode === 'run' ? this.managed.key : undefined; + const variant = this.managed?.variant; + const matches = (spec) => spec.getStatusKey() === key && + (variant === undefined || spec.variant === variant); + const arg = key === undefined ? process.argv[2] : undefined; + if (key !== undefined && ![...this.specs].some(matches)) { + throw new Error(`${key}${variant ?? ''} not found!`); + } if (this.inspectBrk && !arg) { throw new Error('WPT_INSPECT requires a WPT test path'); } for (const spec of this.specs) { + if (key !== undefined && !matches(spec)) continue; if (arg) { if (spec.isSelectedBy(arg)) { queue.push(spec); diff --git a/test/parallel/test-common-wpt-runner.js b/test/parallel/test-common-wpt-runner.js new file mode 100644 index 000000000000..094f1b3ef78f --- /dev/null +++ b/test/parallel/test-common-wpt-runner.js @@ -0,0 +1,133 @@ +'use strict'; + +const common = require('../common'); +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const { spawnSync } = require('child_process'); +const tmpdir = require('../common/tmpdir'); + +if (process.env.NODE_TEST_WPT_REPORT_DIR) { + const { WPTRunner } = require('../common/wpt'); + const runner = new WPTRunner('html/webappapis/atob'); + runner.report.filepath = path.join(process.env.NODE_TEST_WPT_REPORT_DIR, runner.report.filename); + runner.runJsTests(); +} else if (process.env.NODE_TEST_WPT_QUERY_PROBE) { + const { WPTRunner, WPTTestSpec } = require('../common/wpt'); + const runner = new WPTRunner('compression'); + if (!runner.isListing) assert.strictEqual(runner.concurrency, 1); + runner.specs = new Set(['?pass', '?fail'].map((query) => { + const spec = new WPTTestSpec('compression', 'compression-bad-chunks.any.js', [], query, 'window'); + spec.failedTests = ['expected across queries']; + if (process.env.NODE_TEST_WPT_QUERY_PROBE === 'flaky' || + (process.env.NODE_TEST_WPT_QUERY_PROBE === 'mixed' && query === '?pass')) { + spec.flakyTests = [...spec.failedTests]; + } + return spec; + })); + runner.setScriptModifier((script) => { + if (!script.filename.endsWith('compression-bad-chunks.any.js')) return; + script.code = `test(() => assert_true(${process.env.NODE_TEST_WPT_QUERY_PROBE === 'missing'} || + location.search === '?pass'), 'expected across queries');`; + }); + runner.runJsTests(); +} else { + main(); +} + +function main() { + tmpdir.refresh(); + const env = { ...process.env }; + for (const key of ['NODE_TEST_WPT', 'WPT_REPORT', 'WPT_INSPECT']) delete env[key]; + const driver = (name) => path.join(__dirname, '../wpt', `test-${name}.js`); + function invoke(file, config, overrides = {}, status = 0) { + const result = spawnSync(process.execPath, [file], { + env: { ...env, ...overrides, NODE_TEST_WPT: JSON.stringify(config) }, + encoding: 'utf8', timeout: common.platformTimeout(10_000), + maxBuffer: 10 * 1024 * 1024, + }); + assert.ifError(result.error); + assert.strictEqual(result.status, status, result.stdout + result.stderr); + return result.stdout + result.stderr; + } + + function discover(name, overrides, file = driver(name)) { + const stdout = invoke(file, { mode: 'list' }, overrides); + const lines = stdout.split('\n').filter((line) => line.startsWith('NODE_TEST_WPT_MANIFEST:')); + assert.strictEqual(lines.length, 1); + assert.doesNotMatch(stdout, /\[PASS\]/); + const manifest = JSON.parse(lines[0].slice('NODE_TEST_WPT_MANIFEST:'.length)); + assert.strictEqual(manifest.version, 1); + assert.strictEqual(new Set(manifest.tests.map((test) => test.id)).size, manifest.tests.length); + return manifest; + } + + const atob = discover('atob'); + assert.strictEqual(atob.serial, false); + assert.deepStrictEqual(atob.tests.map((test) => test.id), ['base64.any.html', 'base64.any.worker.html']); + const reportRoot = path.join(tmpdir.path, 'reports'); + const canReport = ['darwin', 'linux', 'win32'].includes(process.platform); + if (canReport) fs.mkdirSync(reportRoot, { recursive: true }); + for (const [index, group] of atob.tests.entries()) { + const serial = `group-probe-${process.pid}-${index}`; + const reportPath = path.join(reportRoot, `report-html-webappapis-atob-${serial}.json`); + try { + const stdout = invoke(canReport ? __filename : driver('atob'), + { mode: 'run', source: group.source, key: group.key, variant: group.variant }, canReport ? { + WPT_REPORT: '1', TEST_SERIAL_ID: serial, NODE_TEST_WPT_REPORT_DIR: reportRoot, + } : {}); + const results = stdout.split('\n').filter((line) => line.startsWith('[PASS]')); + assert.ok(results.length > 0); + assert.ok(results.every((line) => line.includes(`${group.id}:`))); + if (canReport) { + const report = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + assert.deepStrictEqual(report.results.map((result) => result.test), + [`/html/webappapis/atob/${group.id}`]); + } + } finally { + fs.rmSync(reportPath, { force: true }); + } + } + + const encoding = discover('encoding'); + const queryGroups = encoding.tests.filter((test) => test.source === 'api-invalid-label.any.js'); + assert.strictEqual(queryGroups.length, 8); + assert.deepStrictEqual([...new Set(queryGroups.map((test) => test.key))], + ['api-invalid-label.any.html', 'api-invalid-label.any.worker.html']); + const timers = discover('timers'); + assert.strictEqual(timers.serial, true); + const skipped = timers.tests.find((test) => test.source === 'negative-settimeout.any.js'); + assert.ok(skipped); + const skippedOutput = invoke(driver('timers'), { mode: 'run', source: skipped.source, key: skipped.key }); + assert.match(skippedOutput, /\[SKIPPED\].*unreliable in Node\.js/); + assert.match(skippedOutput, /1\.\.0 # SKIP/); + assert.doesNotMatch(skippedOutput, /\[PASS\]/); + + if (common.hasSQLite) { + const root = path.join(tmpdir.path, 'discovery'); + const directory = path.join(root, '.tmp.probe'); + fs.mkdirSync(directory, { recursive: true }); + const sentinel = path.join(directory, 'sentinel'); + fs.writeFileSync(sentinel, 'preserved'); + assert.strictEqual(discover('webstorage', { NODE_TEST_DIR: root, TEST_SERIAL_ID: 'probe' }).serial, true); + assert.strictEqual(fs.readFileSync(sentinel, 'utf8'), 'preserved'); + } + + const config = { mode: 'run', source: 'compression-bad-chunks.any.js', key: 'compression-bad-chunks.any.html' }; + for (const probe of ['combined', 'mixed']) { + const strict = discover('compression', { NODE_TEST_WPT_QUERY_PROBE: probe }, __filename); + assert.deepStrictEqual(strict.tests, [{ source: config.source, key: config.key, id: config.key }]); + } + const flaky = discover('compression', { NODE_TEST_WPT_QUERY_PROBE: 'flaky' }, __filename); + assert.deepStrictEqual(flaky.tests.map((test) => test.variant).sort(), ['?fail', '?pass']); + assert.deepStrictEqual(flaky.tests.map((test) => test.id).sort(), + [`${config.key}?fail`, `${config.key}?pass`]); + const single = invoke(__filename, { ...config, variant: '?pass' }, { NODE_TEST_WPT_QUERY_PROBE: 'flaky' }); + assert.match(single, /\.any\.html\?pass:/); + assert.doesNotMatch(single, /\.any\.html\?fail:/); + const combined = invoke(__filename, config, { NODE_TEST_WPT_QUERY_PROBE: 'combined' }); + assert.match(combined, /\.any\.html\?pass:/); + assert.match(combined, /\.any\.html\?fail:/); + const missing = invoke(__filename, config, { NODE_TEST_WPT_QUERY_PROBE: 'missing' }, 1); + assert.match(missing, /Found 2 unexpected passes/); +} diff --git a/test/tools/test_wpt_runner.py b/test/tools/test_wpt_runner.py new file mode 100644 index 000000000000..e57175ea800b --- /dev/null +++ b/test/tools/test_wpt_runner.py @@ -0,0 +1,129 @@ +import contextlib +import json +import os +import sys +import tempfile +import unittest +import warnings +from unittest import mock + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) +sys.path.insert(0, os.path.join(ROOT, 'tools')) +import test as runner + +wpt = runner.get_module('testcfg', os.path.join(ROOT, 'test', 'wpt')) + + +class WPTConfigurationTest(unittest.TestCase): + def setUp(self): + self.stack = contextlib.ExitStack() + self.addCleanup(self.stack.close) + self.stack.enter_context(warnings.catch_warnings()) + # SimpleTestCase's existing source reader does not explicitly close files. + warnings.filterwarnings('ignore', category=ResourceWarning, module='testpy', + message=r'unclosed file .*test-example\.js') + self.root = self.stack.enter_context(tempfile.TemporaryDirectory()) + self.wrapper = os.path.join(self.root, 'test-example.js') + with open(self.wrapper, 'w', encoding='utf8') as source: + source.write('// Flags: --expose-gc\n// Env: GROUP_TEST=kept\n') + self.context = runner.Context( + ROOT, False, sys.executable, [], False, 5, lambda args: args, + False, False, 1, False) + self.config = wpt.GetConfiguration(self.context, self.root) + self.manifest = {'version': 1, 'serial': False, 'tests': [ + {'source': 'nested/a.any.js', 'key': 'nested/a.any.html', 'id': 'nested/a.any.html'}, + {'source': 'nested/a.any.js', 'key': 'nested/a.any.worker.html', + 'id': 'nested/a.any.worker.html'}, + {'source': 'z.any.js', 'key': 'z.any.html', 'id': 'z.any.html'}, + ]} + self.discovery = self.stack.enter_context(mock.patch.object( + runner, 'Execute', side_effect=self.discover)) + + def discover(self, command, context, timeout=None, env=None, **kwargs): + self.assertEqual(json.loads(env['NODE_TEST_WPT']), {'mode': 'list'}) + self.assertEqual(command[-1], self.wrapper) + return runner.CommandOutput( + 0, False, wpt.MANIFEST_PREFIX + json.dumps(self.manifest) + '\n', '') + + def cases(self, selector='wpt/test-example'): + return self.config.ListTests(['wpt'], runner.SplitPath(selector), 'none', 'release') + + def test_discovery_creates_one_case_per_status_group(self): + cases = self.cases() + self.assertEqual([case.group for case in cases], self.manifest['tests']) + self.assertTrue(all(case.parallel for case in cases)) + self.assertEqual([case.GetName() for case in cases], + ['wpt/test-example/' + group['id'] for group in self.manifest['tests']]) + self.assertEqual(len({tuple(case.path) for case in cases}), 3) + self.assertEqual(self.discovery.call_count, 1) + + def test_group_request_preserves_wrapper_startup_flags(self): + self.config.additional_flags = ['--trace-warnings'] + case = self.cases('wpt/test-example/nested/a.any.worker.html')[0] + configuration = case.GetRunConfiguration() + self.assertEqual(configuration['command'], + [sys.executable, '--expose-gc', '--trace-warnings', self.wrapper]) + self.assertEqual(configuration['envs']['GROUP_TEST'], 'kept') + self.assertEqual(json.loads(configuration['envs']['NODE_TEST_WPT']), { + 'mode': 'run', 'source': 'nested/a.any.js', 'key': 'nested/a.any.worker.html', + }) + self.assertEqual(case.GetReportingName(configuration['command']), + 'wpt/test-example/nested/a.any.worker.html') + + def test_group_selection_and_serial_suites(self): + self.manifest['serial'] = True + cases = self.cases('wpt/test-example/nested/*') + self.assertEqual(len(cases), 2) + self.assertTrue(all(not case.parallel for case in cases)) + selected = self.cases('wpt/test-example/nested/a.any.worker.html') + self.assertEqual([case.group['id'] for case in selected], ['nested/a.any.worker.html']) + + def test_variant_queries_are_literal_and_base_selects_all_queries(self): + group = self.manifest['tests'][0] + variants = ['', '?q=[a]+(b)|c', '?q=aaab', '?q=*', '?q=source.js'] + self.manifest['tests'] = [ + {**group, 'id': group['id'] + variant, 'variant': variant} for variant in variants] + base = 'wpt/test-example/' + group['id'] + self.assertEqual([case.group['variant'] for case in self.cases(base)], variants) + for variant in variants[1:]: + with self.subTest(variant=variant): + selector = base + variant + self.assertEqual(runner.NormalizePath(selector), selector) + selected = self.cases(selector) + self.assertEqual(len(selected), 1) + case = selected[0] + self.assertEqual(case.GetName(), selector) + request = json.loads(case.GetRunConfiguration()['envs']['NODE_TEST_WPT']) + self.assertEqual(request, {'mode': 'run', 'source': group['source'], + 'key': group['key'], 'variant': variant}) + empty = self.cases(base)[0] + self.assertEqual(json.loads(empty.GetRunConfiguration()['envs']['NODE_TEST_WPT'])['variant'], '') + + def test_discovery_is_cached_and_uses_existing_directory(self): + absent = os.path.join(self.root, 'not-created') + with mock.patch.dict(os.environ, {'NODE_TEST_DIR': absent}): + self.cases() + self.cases('wpt/test-example/nested/*') + self.assertEqual(self.discovery.call_count, 1) + self.assertFalse(os.path.exists(absent)) + command, _, _, env = self.discovery.call_args.args + self.assertIn('--expose-gc', command) + self.assertEqual(env['NODE_TEST_DIR'], self.root) + + def test_whole_wrapper_feature_skip_keeps_original_test(self): + self.discovery.side_effect = None + self.discovery.return_value = runner.CommandOutput(0, False, '1..0 # Skipped: no feature\n', '') + cases = self.cases() + self.assertEqual(len(cases), 1) + self.assertEqual(cases[0].path, ['wpt', 'test-example']) + self.assertEqual(cases[0].GetRunConfiguration()['command'][-1], self.wrapper) + self.assertNotIn('NODE_TEST_WPT', cases[0].GetRunConfiguration()['envs']) + + def test_malformed_discovery_is_an_error(self): + self.manifest['tests'] = 'not a list' + with self.assertRaisesRegex(RuntimeError, 'WPT discovery failed'): + self.cases() + + +if __name__ == '__main__': + unittest.main() diff --git a/test/wpt/README.md b/test/wpt/README.md index ac96e3b247ba..1b6911740a06 100644 --- a/test/wpt/README.md +++ b/test/wpt/README.md @@ -20,6 +20,15 @@ Run a WPT module through the Python test runner: tools/test.py wpt/test-url ``` +Select a generated path from the Python runner's output: + +```bash +tools/test.py 'wpt/test-webcrypto/derive_bits_keys/hkdf.https.any.worker.html?1-1000' +``` + +Omit the query string to select all its variants. Variants sharing strict +expected-failure rules run together. + Pass a source file to its module runner to run all globals and variants generated from that file: diff --git a/test/wpt/test-compression.js b/test/wpt/test-compression.js index 404cb23687ca..23800734922a 100644 --- a/test/wpt/test-compression.js +++ b/test/wpt/test-compression.js @@ -2,8 +2,7 @@ const { WPTRunner } = require('../common/wpt'); -// Runs each spec in its own process; this suite has crashed the runner in CI. -const runner = new WPTRunner('compression', { backend: 'process' }); +const runner = new WPTRunner('compression'); runner.pretendGlobalThisAs('Window'); diff --git a/test/wpt/test-streams.js b/test/wpt/test-streams.js index e9d23348db07..71c25fbd56b2 100644 --- a/test/wpt/test-streams.js +++ b/test/wpt/test-streams.js @@ -2,8 +2,7 @@ const { WPTRunner } = require('../common/wpt'); -// Runs each spec in its own process; this suite has crashed the runner in CI. -const runner = new WPTRunner('streams', { backend: 'process' }); +const runner = new WPTRunner('streams'); // Set a script that will be executed in the worker before running the tests. runner.pretendGlobalThisAs('Window'); diff --git a/test/wpt/test-wasm-jsapi.mjs b/test/wpt/test-wasm-jsapi.mjs index 67050e67cf7b..4e53da5bdd88 100644 --- a/test/wpt/test-wasm-jsapi.mjs +++ b/test/wpt/test-wasm-jsapi.mjs @@ -15,8 +15,7 @@ try { } if (supportsSimd) { - // Runs each spec in its own process; this suite has crashed the runner in CI. - const runner = new WPTRunner('wasm/jsapi', { backend: 'process' }); + const runner = new WPTRunner('wasm/jsapi'); runner.setFlags(['--experimental-wasm-modules']); runner.runJsTests(); diff --git a/test/wpt/test-webcrypto.js b/test/wpt/test-webcrypto.js index a435d429af9f..409b94451ba1 100644 --- a/test/wpt/test-webcrypto.js +++ b/test/wpt/test-webcrypto.js @@ -6,8 +6,7 @@ if (!common.hasCrypto) const { WPTRunner } = require('../common/wpt'); -// Runs each spec in its own process; this suite has crashed the runner in CI. -const runner = new WPTRunner('WebCryptoAPI', { backend: 'process' }); +const runner = new WPTRunner('WebCryptoAPI'); // Experimental warnings drown out the actual test output. runner.setFlags(['--disable-warning=ExperimentalWarning']); diff --git a/test/wpt/test-webstorage.js b/test/wpt/test-webstorage.js index a677659f4b04..b0cc3d6bf13d 100644 --- a/test/wpt/test-webstorage.js +++ b/test/wpt/test-webstorage.js @@ -6,7 +6,7 @@ const { WPTRunner } = require('../common/wpt'); const { join } = require('node:path'); const runner = new WPTRunner('webstorage', { concurrency: 1 }); -tmpdir.refresh(); +if (!runner.isListing) tmpdir.refresh(); runner.setFlags([ '--localstorage-file', join(tmpdir.path, 'wpt-tests.localstorage'), diff --git a/test/wpt/testcfg.py b/test/wpt/testcfg.py index 3c356cf474d8..c771150b1637 100644 --- a/test/wpt/testcfg.py +++ b/test/wpt/testcfg.py @@ -1,6 +1,105 @@ -import sys, os +import json +import os +import re +import sys + sys.path.append(os.path.join(os.path.dirname(__file__), '..')) import testpy +import test + + +MANIFEST_PREFIX = 'NODE_TEST_WPT_MANIFEST:' + + +class WPTTestCase(testpy.SimpleTestCase): + def __init__(self, path, file, arch, mode, context, config, group, serial): + super(WPTTestCase, self).__init__( + path, file, arch, mode, context, config, config.additional_flags) + self.group = group + self.parallel = not serial + + def GetName(self): + return '/'.join(self.path) + + def GetReportingName(self, command): + return self.GetName() + + def GetRunConfiguration(self): + configuration = super(WPTTestCase, self).GetRunConfiguration() + request = { + 'mode': 'run', 'source': self.group['source'], 'key': self.group['key'], + } + if 'variant' in self.group: + request['variant'] = self.group['variant'] + configuration['envs']['NODE_TEST_WPT'] = json.dumps(request) + return configuration + + +class WPTTestConfiguration(testpy.SimpleTestConfiguration): + def __init__(self, context, root): + super(WPTTestConfiguration, self).__init__(context, root, 'wpt') + self.manifests = {} + + def _Discover(self, wrapper): + key = (wrapper.file, wrapper.arch, wrapper.mode) + if key in self.manifests: + return self.manifests[key] + configuration = wrapper.GetRunConfiguration() + configuration['envs']['NODE_TEST_WPT'] = json.dumps({'mode': 'list'}) + # common/tmpdir resolves this path before Main creates the execution dir. + configuration['envs']['NODE_TEST_DIR'] = os.path.abspath(self.root) + output = test.Execute( + self.context.processor(configuration['command']), self.context, + self.context.GetTimeout(wrapper.mode), configuration['envs']) + if output.exit_code != 0 or output.timed_out: + raise RuntimeError('WPT discovery failed for %s:\n%s%s' % ( + wrapper.file, output.stdout, output.stderr)) + manifests = [line[len(MANIFEST_PREFIX):] for line in output.stdout.splitlines() + if line.startswith(MANIFEST_PREFIX)] + if not manifests and test.skip_regex.search(output.stdout): + self.manifests[key] = None + return None + try: + if len(manifests) != 1: + raise ValueError('expected exactly one WPT manifest') + manifest = json.loads(manifests[0]) + if (manifest.get('version') != 1 or + not isinstance(manifest.get('serial'), bool) or + not isinstance(manifest.get('tests'), list) or + any(not isinstance(group.get(field), str) + for group in manifest['tests'] for field in ['source', 'key', 'id']) or + any('variant' in group and not isinstance(group['variant'], str) + for group in manifest['tests'])): + raise ValueError('invalid WPT manifest') + except (AttributeError, TypeError, ValueError) as error: + raise RuntimeError('WPT discovery failed for %s: %s' % ( + wrapper.file, error)) from error + self.manifests[key] = manifest + return manifest + + def ListTests(self, current_path, path, arch, mode): + wrappers = super(WPTTestConfiguration, self).ListTests( + current_path, path[:2], arch, mode) + selector = '/'.join(part.pattern for part in path[2:]) + pattern = re.escape(selector).replace(r'\*', '.*') + r'(?:/.*)?' + result = [] + for wrapper in wrappers: + manifest = self._Discover(wrapper) + if manifest is None: + result.append(wrapper) + continue + for group in manifest['tests']: + case_path = wrapper.path + group['id'].split('/') + # Query selectors are literal; a query-free path selects all its variants. + if '?' in selector: + selected = group['id'] == selector + else: + selected = not selector or re.fullmatch(pattern, group['id'].split('?')[0]) + if selected: + result.append(WPTTestCase(case_path, wrapper.file, arch, mode, + self.context, self, group, manifest['serial'])) + return result + def GetConfiguration(context, root): - return testpy.SimpleTestConfiguration(context, root, 'wpt') + return WPTTestConfiguration(context, root) diff --git a/tools/test.py b/tools/test.py index 2c2a4d78d80a..2dfc3e105ac8 100755 --- a/tools/test.py +++ b/tools/test.py @@ -360,9 +360,7 @@ def HasRun(self, output): # Print test name as (for example) "parallel/test-assert". Tests that are # scraped from the addons documentation are all named test.js, making it # hard to decipher what test is running when only the filename is printed. - prefix = abspath(join(dirname(__file__), '../test')) + os.sep - command = output.command[-1] - command = NormalizePath(command, prefix) + command = output.test.GetReportingName(output.command) if output.UnexpectedOutput(): status_line = 'not ok %i %s' % (self._done, command) @@ -573,6 +571,10 @@ def __init__(self, context, path, arch, mode): self.serial_id = 0 self.thread_id = 0 + def GetReportingName(self, command): + prefix = abspath(join(dirname(__file__), '../test')) + os.sep + return NormalizePath(command[-1], prefix) + def IsNegative(self): return self.context.expect_fail @@ -1544,6 +1546,8 @@ def NormalizePath(path, prefix='test/'): path = path.replace('\\', '/') if path.startswith(prefix): path = path[len(prefix):] + if '?' in path or '#' in path: + return path if path.endswith('.js'): path = path[:-3] elif path.endswith('.mjs'): @@ -1799,7 +1803,8 @@ def Main(): sys.exit(1) def should_keep(case): - if any((s in case.file) for s in options.skip_tests): + if any(s in case.file or s in '/'.join(case.path) + for s in options.skip_tests): return False elif SKIP in case.outcomes: return False @@ -1825,7 +1830,7 @@ def should_keep(case): # Must ensure the list of tests is sorted before selecting, to avoid # silent errors if this file is changed to list the tests in a way that # can be different in different machines - cases_to_run.sort(key=lambda c: (c.arch, c.mode, c.file)) + cases_to_run.sort(key=lambda c: (c.arch, c.mode, c.file, c.path)) cases_to_run = [ cases_to_run[i] for i in range(options.run[0], len(cases_to_run), From 88a00f118fc0ef45a90c467f07442539ece02a3a Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 11 Sep 2026 17:35:33 +0200 Subject: [PATCH 2/2] fixup! test: schedule WPT variants individually --- test/common/wpt.js | 13 ++++--- test/parallel/test-common-wpt-runner.js | 18 ++++++++- test/tools/test_wpt_runner.py | 49 +++++++++++++++++++++++-- test/wpt/testcfg.py | 11 +++++- tools/test.py | 13 ++++--- 5 files changed, 86 insertions(+), 18 deletions(-) diff --git a/test/common/wpt.js b/test/common/wpt.js index 69c93b1cbd2e..e2668d75da7d 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -461,12 +461,14 @@ class WPTTestSpec { /** * Whether a command line argument selects this spec. Accepts the source file * name, which selects every global and variant generated from it, or a test - * path as printed alongside the results, which selects only this one. + * path as printed alongside the results. Omitting its query selects all + * variants of that global. * @param {string} arg * @returns {boolean} */ isSelectedBy(arg) { - if (arg === this.getTestPath()) { + const testPath = this.getTestPath(); + if (arg === testPath || arg === testPath.split('?')[0]) { return true; } const [filename, variant = ''] = arg.split('?'); @@ -829,7 +831,7 @@ class WPTRunner { (['source', 'key'].some((key) => typeof this.managed[key] !== 'string' || !this.managed[key]) || (this.managed.variant !== undefined && typeof this.managed.variant !== 'string')))) { - throw new Error('Invalid NODE_TEST_WPT configuration'); + throw new Error('Invalid WPT runner configuration'); } } this.isListing = this.managed?.mode === 'list'; @@ -987,12 +989,13 @@ class WPTRunner { const grouped = specs.some((spec) => spec.failedTests.some((name) => isUnexpectedPass(spec, name))); return (grouped ? [specs[0]] : specs).map((spec) => { - const id = spec.getTestPath().slice(this.path.length + 1); + const selector = grouped ? spec.getTestPath().split('?')[0] : spec.getTestPath(); return { source: spec.filename.split(path.sep).join('/'), key: spec.getStatusKey(), ...(grouped ? {} : { variant: spec.variant }), - id: grouped ? id.split('?')[0] : id, + id: selector.slice(this.path.length + 1), + selector, }; }); }); diff --git a/test/parallel/test-common-wpt-runner.js b/test/parallel/test-common-wpt-runner.js index 094f1b3ef78f..ea41caa6cefc 100644 --- a/test/parallel/test-common-wpt-runner.js +++ b/test/parallel/test-common-wpt-runner.js @@ -15,7 +15,7 @@ if (process.env.NODE_TEST_WPT_REPORT_DIR) { } else if (process.env.NODE_TEST_WPT_QUERY_PROBE) { const { WPTRunner, WPTTestSpec } = require('../common/wpt'); const runner = new WPTRunner('compression'); - if (!runner.isListing) assert.strictEqual(runner.concurrency, 1); + if (runner.managed?.mode === 'run') assert.strictEqual(runner.concurrency, 1); runner.specs = new Set(['?pass', '?fail'].map((query) => { const spec = new WPTTestSpec('compression', 'compression-bad-chunks.any.js', [], query, 'window'); spec.failedTests = ['expected across queries']; @@ -116,7 +116,9 @@ function main() { const config = { mode: 'run', source: 'compression-bad-chunks.any.js', key: 'compression-bad-chunks.any.html' }; for (const probe of ['combined', 'mixed']) { const strict = discover('compression', { NODE_TEST_WPT_QUERY_PROBE: probe }, __filename); - assert.deepStrictEqual(strict.tests, [{ source: config.source, key: config.key, id: config.key }]); + assert.deepStrictEqual(strict.tests, [{ + source: config.source, key: config.key, id: config.key, selector: `compression/${config.key}`, + }]); } const flaky = discover('compression', { NODE_TEST_WPT_QUERY_PROBE: 'flaky' }, __filename); assert.deepStrictEqual(flaky.tests.map((test) => test.variant).sort(), ['?fail', '?pass']); @@ -130,4 +132,16 @@ function main() { assert.match(combined, /\.any\.html\?fail:/); const missing = invoke(__filename, config, { NODE_TEST_WPT_QUERY_PROBE: 'missing' }, 1); assert.match(missing, /Found 2 unexpected passes/); + + for (const query of ['', '?fail']) { + const direct = spawnSync(process.execPath, [__filename, `compression/${config.key}${query}`], { + env: { ...env, NODE_TEST_WPT_QUERY_PROBE: 'combined' }, + encoding: 'utf8', timeout: common.platformTimeout(10_000), + }); + assert.ifError(direct.error); + assert.strictEqual(direct.status, 0, direct.stdout + direct.stderr); + assert.match(direct.stdout, /\.any\.html\?fail:/); + if (query) assert.doesNotMatch(direct.stdout, /\.any\.html\?pass:/); + else assert.match(direct.stdout, /\.any\.html\?pass:/); + } } diff --git a/test/tools/test_wpt_runner.py b/test/tools/test_wpt_runner.py index e57175ea800b..5897e396cb4f 100644 --- a/test/tools/test_wpt_runner.py +++ b/test/tools/test_wpt_runner.py @@ -1,6 +1,7 @@ import contextlib import json import os +import shlex import sys import tempfile import unittest @@ -31,10 +32,11 @@ def setUp(self): False, False, 1, False) self.config = wpt.GetConfiguration(self.context, self.root) self.manifest = {'version': 1, 'serial': False, 'tests': [ - {'source': 'nested/a.any.js', 'key': 'nested/a.any.html', 'id': 'nested/a.any.html'}, + {'source': 'nested/a.any.js', 'key': 'nested/a.any.html', 'id': 'nested/a.any.html', + 'selector': 'example/nested/a.any.html'}, {'source': 'nested/a.any.js', 'key': 'nested/a.any.worker.html', - 'id': 'nested/a.any.worker.html'}, - {'source': 'z.any.js', 'key': 'z.any.html', 'id': 'z.any.html'}, + 'id': 'nested/a.any.worker.html', 'selector': 'example/nested/a.any.worker.html'}, + {'source': 'z.any.js', 'key': 'z.any.html', 'id': 'z.any.html', 'selector': 'example/z.any.html'}, ]} self.discovery = self.stack.enter_context(mock.patch.object( runner, 'Execute', side_effect=self.discover)) @@ -82,7 +84,8 @@ def test_variant_queries_are_literal_and_base_selects_all_queries(self): group = self.manifest['tests'][0] variants = ['', '?q=[a]+(b)|c', '?q=aaab', '?q=*', '?q=source.js'] self.manifest['tests'] = [ - {**group, 'id': group['id'] + variant, 'variant': variant} for variant in variants] + {**group, 'id': group['id'] + variant, 'selector': group['selector'] + variant, + 'variant': variant} for variant in variants] base = 'wpt/test-example/' + group['id'] self.assertEqual([case.group['variant'] for case in self.cases(base)], variants) for variant in variants[1:]: @@ -124,6 +127,44 @@ def test_malformed_discovery_is_an_error(self): with self.assertRaisesRegex(RuntimeError, 'WPT discovery failed'): self.cases() + def test_failure_commands_preserve_actual_command_and_selection(self): + self.config.additional_flags = ["--title=space ' $|?", '--trace-warnings'] + query = "?q=space ' | $(echo)" + for group in self.manifest['tests'][:2]: + group.update(id=group['id'] + query, selector=group['selector'] + query, variant=query) + self.manifest['tests'].append({'source': 'empty.any.js', 'key': 'empty.any.html', + 'id': 'empty.any.html', 'selector': 'example/empty.any.html', 'variant': ''}) + cases = self.cases() + self.assertEqual(len(cases), 4) + self.context.processor = lambda args: ['valgrind', '--tool=memcheck', *args, 'suffix with |'] + with mock.patch.object(sys, 'platform', 'linux'): + for case, group in zip(cases, self.manifest['tests']): + command = case.GetRunConfiguration()['command'] + expected = [sys.executable, '--expose-gc', "--title=space ' $|?", '--trace-warnings', + self.wrapper] + self.assertEqual(command, expected) + command = self.context.processor(command) + rerun = ['valgrind', '--tool=memcheck', *expected, group['selector'], 'suffix with |'] + self.assertEqual(shlex.split(case.GetFailureCommand(command)), rerun) + failure = runner.TestOutput(case, command, + runner.CommandOutput(1, False, '', 'probe failure'), False) + printer = runner.ProgressIndicator([], runner.RUN, 0) + self.assertIn('Command: ' + shlex.join(rerun), printer.GetFailureOutput(failure)) + + def test_failure_command_quotes_powershell_metacharacters(self): + case = self.cases()[0] + case.file = 'test driver.js' + case.group['selector'] = "example/a.any.html?q=O'Brien|x" + command = ['node', case.file] + with mock.patch.object(sys, 'platform', 'win32'): + self.assertEqual(case.GetFailureCommand(command), + "& 'node' 'test driver.js' 'example/a.any.html?q=O''Brien|x'") + + def test_ordinary_failure_command_is_unchanged(self): + case = runner.TestCase(self.context, ['parallel', 'test-example'], 'none', 'release') + command = [sys.executable, '--expose-gc', 'test/parallel/path with spaces.js'] + self.assertEqual(case.GetFailureCommand(command), runner.EscapeCommand(command)) + if __name__ == '__main__': unittest.main() diff --git a/test/wpt/testcfg.py b/test/wpt/testcfg.py index c771150b1637..2f3097da96df 100644 --- a/test/wpt/testcfg.py +++ b/test/wpt/testcfg.py @@ -1,6 +1,7 @@ import json import os import re +import shlex import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..')) @@ -24,6 +25,14 @@ def GetName(self): def GetReportingName(self, command): return self.GetName() + def GetFailureCommand(self, command): + command = list(command) + command.insert(command.index(self.file) + 1, self.group['selector']) + if sys.platform == 'win32': + # PowerShell quoting also protects query metacharacters such as | and &. + return '& ' + ' '.join("'" + arg.replace("'", "''") + "'" for arg in command) + return shlex.join(command) + def GetRunConfiguration(self): configuration = super(WPTTestCase, self).GetRunConfiguration() request = { @@ -67,7 +76,7 @@ def _Discover(self, wrapper): not isinstance(manifest.get('serial'), bool) or not isinstance(manifest.get('tests'), list) or any(not isinstance(group.get(field), str) - for group in manifest['tests'] for field in ['source', 'key', 'id']) or + for group in manifest['tests'] for field in ['source', 'key', 'id', 'selector']) or any('variant' in group and not isinstance(group['variant'], str) for group in manifest['tests'])): raise ValueError('invalid WPT manifest') diff --git a/tools/test.py b/tools/test.py index 2dfc3e105ac8..5c06bfff6b42 100755 --- a/tools/test.py +++ b/tools/test.py @@ -126,7 +126,7 @@ def GetFailureOutput(self, failure): if failure.output.stdout: output += ["--- stdout ---"] output += [failure.output.stdout.strip()] - output += ["Command: %s" % EscapeCommand(failure.command)] + output += ["Command: %s" % failure.test.GetFailureCommand(failure.command)] if failure.HasCrashed(): output += ["--- %s ---" % PrintCrashed(failure.output.exit_code)] if failure.HasTimedOut(): @@ -423,9 +423,7 @@ def HasRun(self, output): # Print test name as (for example) "parallel/test-assert". Tests that are # scraped from the addons documentation are all named test.js, making it # hard to decipher what test is running when only the filename is printed. - prefix = abspath(join(dirname(__file__), '../test')) + os.sep - command = output.command[-1] - command = NormalizePath(command, prefix) + command = output.test.GetReportingName(output.command) stdout = output.output.stdout.strip() printed_file = False @@ -471,7 +469,7 @@ def HasRun(self, output): stderr = output.output.stderr.strip() if len(stderr): print(self.templates['stderr'] % stderr) - print("Command: %s" % EscapeCommand(output.command)) + print("Command: %s" % output.test.GetFailureCommand(output.command)) if output.HasCrashed(): print("--- %s ---" % PrintCrashed(output.output.exit_code)) if output.HasTimedOut(): @@ -575,6 +573,9 @@ def GetReportingName(self, command): prefix = abspath(join(dirname(__file__), '../test')) + os.sep return NormalizePath(command[-1], prefix) + def GetFailureCommand(self, command): + return EscapeCommand(command) + def IsNegative(self): return self.context.expect_fail @@ -1864,7 +1865,7 @@ def should_keep(case): elif result['failed']: print("\nFailed tests:") for failure in result['failed']: - print(EscapeCommand(failure.command)) + print(failure.test.GetFailureCommand(failure.command)) else: print("\nTest aborted.") return exitcode