test: schedule WPT variants individually

test/common/wpt.js

@@ -96,8 +96,8 @@ class ReportResult {
 // Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation
 // for more details.
 class WPTReport {
-  constructor(testPath) {
-    this.filename = `report-${testPath.replaceAll('/', '-')}.json`;
+  constructor(testPath, suffix = '') {
+    this.filename = `report-${testPath.replaceAll('/', '-')}${suffix}.json`;
     this.filepath = path.join(__dirname, `../../out/wpt/${this.filename}`);
     /** @type {Map<string, ReportResult>} */
     this.results = new Map();
@@ -461,12 +461,14 @@ class WPTTestSpec {
   /**
    * Whether a command line argument selects this spec. Accepts the source file
    * name, which selects every global and variant generated from it, or a test
-   * path as printed alongside the results, which selects only this one.
+   * path as printed alongside the results. Omitting its query selects all
+   * variants of that global.
    * @param {string} arg
    * @returns {boolean}
    */
   isSelectedBy(arg) {
-    if (arg === this.getTestPath()) {
+    const testPath = this.getTestPath();
+    if (arg === testPath || arg === testPath.split('?')[0]) {
       return true;
     }
     const [filename, variant = ''] = arg.split('?');
@@ -608,7 +610,7 @@ class StatusLoader {
     return result;
   }
 
-  load() {
+  load(source) {
     const dir = path.join(__dirname, '..', 'wpt');
     let result;
 
@@ -625,7 +627,7 @@ class StatusLoader {
     this.rules.addRules(result);
 
     const subDir = fixtures.path('wpt', this.path);
-    const list = this.grep(subDir);
+    const list = source === undefined ? this.grep(subDir) : [path.join(subDir, source)];
     for (const file of list) {
       const relativePath = path.relative(subDir, file);
       const match = this.rules.match(relativePath);
@@ -813,14 +815,29 @@ const backends = {
 };
 
 class WPTRunner {
-  constructor(path, {
-    concurrency = os.availableParallelism() - 1 || 1,
-    backend = 'thread',
-  } = {}) {
+  constructor(path, options = {}) {
+    let {
+      concurrency = os.availableParallelism() - 1 || 1,
+      backend = 'thread',
+    } = options;
     if (!Number.isInteger(concurrency) || concurrency < 1) {
       throw new TypeError('WPT concurrency must be a positive integer');
     }
 
+    if (process.env.NODE_TEST_WPT !== undefined) {
+      this.managed = JSON.parse(process.env.NODE_TEST_WPT);
+      if (!this.managed || !['list', 'run'].includes(this.managed.mode) ||
+          (this.managed.mode === 'run' &&
+           (['source', 'key'].some((key) =>
+             typeof this.managed[key] !== 'string' || !this.managed[key]) ||
+            (this.managed.variant !== undefined && typeof this.managed.variant !== 'string')))) {
+        throw new Error('Invalid WPT runner configuration');
+      }
+    }
+    this.isListing = this.managed?.mode === 'list';
+    this.serial = options.concurrency === 1;
+    if (this.managed?.mode === 'run') concurrency = 1;
+
     // RISC-V has very limited virtual address space in the currently common
     // sv39 mode, in which we can only create a very limited number of wasm
     // memories(27 from a fresh node repl). Limit the concurrency to avoid
@@ -860,7 +877,7 @@ class WPTRunner {
     this.initScript = null;
 
     this.status = new StatusLoader(path);
-    this.status.load();
+    this.status.load(this.managed?.mode === 'run' ? this.managed.source : undefined);
     this.statusFile = this.status.statusFile;
     this.specs = new Set(this.status.specs);
 
@@ -872,8 +889,9 @@ class WPTRunner {
 
     this.subtestCounts = { passed: 0, failed: 0, expectedFailures: 0, skipped: 0, unexpectedPasses: 0 };
 
-    if (process.env.WPT_REPORT != null) {
-      this.report = new WPTReport(path);
+    if (process.env.WPT_REPORT != null && !this.isListing) {
+      const suffix = this.managed ? `-${process.env.TEST_SERIAL_ID || process.pid}` : '';
+      this.report = new WPTReport(path, suffix);
     }
   }
 
@@ -958,7 +976,40 @@ class WPTRunner {
   // TODO(joyeecheung): work with the upstream to port more tests in .html
   // to .js.
   async runJsTests() {
+    if (this.isListing) {
+      const groups = new Map();
+      for (const spec of this.specs) {
+        const key = spec.getStatusKey();
+        if (!groups.has(key)) groups.set(key, []);
+        groups.get(key).push(spec);
+      }
+      const tests = [...groups.values()].flatMap((specs) => {
+        // Strict expected failures are checked across all query variants.
+        // Keep that group together; all other variants are independent tasks.
+        const grouped = specs.some((spec) =>
+          spec.failedTests.some((name) => isUnexpectedPass(spec, name)));
+        return (grouped ? [specs[0]] : specs).map((spec) => {
+          const selector = grouped ? spec.getTestPath().split('?')[0] : spec.getTestPath();
+          return {
+            source: spec.filename.split(path.sep).join('/'),
+            key: spec.getStatusKey(),
+            ...(grouped ? {} : { variant: spec.variant }),
+            id: selector.slice(this.path.length + 1),
+            selector,
+          };
+        });
+      });
+      console.log(`NODE_TEST_WPT_MANIFEST:${JSON.stringify({
+        version: 1,
+        serial: this.serial,
+        tests: tests.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)),
+      })}`);
+      return;
+    }
     const queue = this.buildQueue();
+    if (this.managed && queue.length === 0) {
+      console.log('1..0 # SKIP No runnable WPT variants');
+    }
 
     const run = limit(this.concurrency);
     const jobs = [];
@@ -1313,11 +1364,19 @@ class WPTRunner {
   buildQueue() {
     const queue = [];
     this.skippedSpecCount = 0;
-    const arg = process.argv[2];
+    const key = this.managed?.mode === 'run' ? this.managed.key : undefined;
+    const variant = this.managed?.variant;
+    const matches = (spec) => spec.getStatusKey() === key &&
+      (variant === undefined || spec.variant === variant);
+    const arg = key === undefined ? process.argv[2] : undefined;
+    if (key !== undefined && ![...this.specs].some(matches)) {
+      throw new Error(`${key}${variant ?? ''} not found!`);
+    }
     if (this.inspectBrk && !arg) {
       throw new Error('WPT_INSPECT requires a WPT test path');
     }
     for (const spec of this.specs) {
+      if (key !== undefined && !matches(spec)) continue;
       if (arg) {
         if (spec.isSelectedBy(arg)) {
           queue.push(spec);

test/parallel/test-common-wpt-runner.js

@@ -0,0 +1,147 @@
+'use strict';
+
+const common = require('../common');
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const tmpdir = require('../common/tmpdir');
+
+if (process.env.NODE_TEST_WPT_REPORT_DIR) {
+  const { WPTRunner } = require('../common/wpt');
+  const runner = new WPTRunner('html/webappapis/atob');
+  runner.report.filepath = path.join(process.env.NODE_TEST_WPT_REPORT_DIR, runner.report.filename);
+  runner.runJsTests();
+} else if (process.env.NODE_TEST_WPT_QUERY_PROBE) {
+  const { WPTRunner, WPTTestSpec } = require('../common/wpt');
+  const runner = new WPTRunner('compression');
+  if (runner.managed?.mode === 'run') assert.strictEqual(runner.concurrency, 1);
+  runner.specs = new Set(['?pass', '?fail'].map((query) => {
+    const spec = new WPTTestSpec('compression', 'compression-bad-chunks.any.js', [], query, 'window');
+    spec.failedTests = ['expected across queries'];
+    if (process.env.NODE_TEST_WPT_QUERY_PROBE === 'flaky' ||
+        (process.env.NODE_TEST_WPT_QUERY_PROBE === 'mixed' && query === '?pass')) {
+      spec.flakyTests = [...spec.failedTests];
+    }
+    return spec;
+  }));
+  runner.setScriptModifier((script) => {
+    if (!script.filename.endsWith('compression-bad-chunks.any.js')) return;
+    script.code = `test(() => assert_true(${process.env.NODE_TEST_WPT_QUERY_PROBE === 'missing'} ||
+      location.search === '?pass'), 'expected across queries');`;
+  });
+  runner.runJsTests();
+} else {
+  main();
+}
+
+function main() {
+  tmpdir.refresh();
+  const env = { ...process.env };
+  for (const key of ['NODE_TEST_WPT', 'WPT_REPORT', 'WPT_INSPECT']) delete env[key];
+  const driver = (name) => path.join(__dirname, '../wpt', `test-${name}.js`);
+  function invoke(file, config, overrides = {}, status = 0) {
+    const result = spawnSync(process.execPath, [file], {
+      env: { ...env, ...overrides, NODE_TEST_WPT: JSON.stringify(config) },
+      encoding: 'utf8', timeout: common.platformTimeout(10_000),
+      maxBuffer: 10 * 1024 * 1024,
+    });
+    assert.ifError(result.error);
+    assert.strictEqual(result.status, status, result.stdout + result.stderr);
+    return result.stdout + result.stderr;
+  }
+
+  function discover(name, overrides, file = driver(name)) {
+    const stdout = invoke(file, { mode: 'list' }, overrides);
+    const lines = stdout.split('\n').filter((line) => line.startsWith('NODE_TEST_WPT_MANIFEST:'));
+    assert.strictEqual(lines.length, 1);
+    assert.doesNotMatch(stdout, /\[PASS\]/);
+    const manifest = JSON.parse(lines[0].slice('NODE_TEST_WPT_MANIFEST:'.length));
+    assert.strictEqual(manifest.version, 1);
+    assert.strictEqual(new Set(manifest.tests.map((test) => test.id)).size, manifest.tests.length);
+    return manifest;
+  }
+
+  const atob = discover('atob');
+  assert.strictEqual(atob.serial, false);
+  assert.deepStrictEqual(atob.tests.map((test) => test.id), ['base64.any.html', 'base64.any.worker.html']);
+  const reportRoot = path.join(tmpdir.path, 'reports');
+  const canReport = ['darwin', 'linux', 'win32'].includes(process.platform);
+  if (canReport) fs.mkdirSync(reportRoot, { recursive: true });
+  for (const [index, group] of atob.tests.entries()) {
+    const serial = `group-probe-${process.pid}-${index}`;
+    const reportPath = path.join(reportRoot, `report-html-webappapis-atob-${serial}.json`);
+    try {
+      const stdout = invoke(canReport ? __filename : driver('atob'),
+                            { mode: 'run', source: group.source, key: group.key, variant: group.variant }, canReport ? {
+                              WPT_REPORT: '1', TEST_SERIAL_ID: serial, NODE_TEST_WPT_REPORT_DIR: reportRoot,
+                            } : {});
+      const results = stdout.split('\n').filter((line) => line.startsWith('[PASS]'));
+      assert.ok(results.length > 0);
+      assert.ok(results.every((line) => line.includes(`${group.id}:`)));
+      if (canReport) {
+        const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
+        assert.deepStrictEqual(report.results.map((result) => result.test),
+                               [`/html/webappapis/atob/${group.id}`]);
+      }
+    } finally {
+      fs.rmSync(reportPath, { force: true });
+    }
+  }
+
+  const encoding = discover('encoding');
+  const queryGroups = encoding.tests.filter((test) => test.source === 'api-invalid-label.any.js');
+  assert.strictEqual(queryGroups.length, 8);
+  assert.deepStrictEqual([...new Set(queryGroups.map((test) => test.key))],
+                         ['api-invalid-label.any.html', 'api-invalid-label.any.worker.html']);
+  const timers = discover('timers');
+  assert.strictEqual(timers.serial, true);
+  const skipped = timers.tests.find((test) => test.source === 'negative-settimeout.any.js');
+  assert.ok(skipped);
+  const skippedOutput = invoke(driver('timers'), { mode: 'run', source: skipped.source, key: skipped.key });
+  assert.match(skippedOutput, /\[SKIPPED\].*unreliable in Node\.js/);
+  assert.match(skippedOutput, /1\.\.0 # SKIP/);
+  assert.doesNotMatch(skippedOutput, /\[PASS\]/);
+
+  if (common.hasSQLite) {
+    const root = path.join(tmpdir.path, 'discovery');
+    const directory = path.join(root, '.tmp.probe');
+    fs.mkdirSync(directory, { recursive: true });
+    const sentinel = path.join(directory, 'sentinel');
+    fs.writeFileSync(sentinel, 'preserved');
+    assert.strictEqual(discover('webstorage', { NODE_TEST_DIR: root, TEST_SERIAL_ID: 'probe' }).serial, true);
+    assert.strictEqual(fs.readFileSync(sentinel, 'utf8'), 'preserved');
+  }
+
+  const config = { mode: 'run', source: 'compression-bad-chunks.any.js', key: 'compression-bad-chunks.any.html' };
+  for (const probe of ['combined', 'mixed']) {
+    const strict = discover('compression', { NODE_TEST_WPT_QUERY_PROBE: probe }, __filename);
+    assert.deepStrictEqual(strict.tests, [{
+      source: config.source, key: config.key, id: config.key, selector: `compression/${config.key}`,
+    }]);
+  }
+  const flaky = discover('compression', { NODE_TEST_WPT_QUERY_PROBE: 'flaky' }, __filename);
+  assert.deepStrictEqual(flaky.tests.map((test) => test.variant).sort(), ['?fail', '?pass']);
+  assert.deepStrictEqual(flaky.tests.map((test) => test.id).sort(),
+                         [`${config.key}?fail`, `${config.key}?pass`]);
+  const single = invoke(__filename, { ...config, variant: '?pass' }, { NODE_TEST_WPT_QUERY_PROBE: 'flaky' });
+  assert.match(single, /\.any\.html\?pass:/);
+  assert.doesNotMatch(single, /\.any\.html\?fail:/);
+  const combined = invoke(__filename, config, { NODE_TEST_WPT_QUERY_PROBE: 'combined' });
+  assert.match(combined, /\.any\.html\?pass:/);
+  assert.match(combined, /\.any\.html\?fail:/);
+  const missing = invoke(__filename, config, { NODE_TEST_WPT_QUERY_PROBE: 'missing' }, 1);
+  assert.match(missing, /Found 2 unexpected passes/);
+
+  for (const query of ['', '?fail']) {
+    const direct = spawnSync(process.execPath, [__filename, `compression/${config.key}${query}`], {
+      env: { ...env, NODE_TEST_WPT_QUERY_PROBE: 'combined' },
+      encoding: 'utf8', timeout: common.platformTimeout(10_000),
+    });
+    assert.ifError(direct.error);
+    assert.strictEqual(direct.status, 0, direct.stdout + direct.stderr);
+    assert.match(direct.stdout, /\.any\.html\?fail:/);
+    if (query) assert.doesNotMatch(direct.stdout, /\.any\.html\?pass:/);
+    else assert.match(direct.stdout, /\.any\.html\?pass:/);
+  }
+}