feat(test): add retryStrategy option to defer retries

docs/src/test-api/class-testconfig.md

@@ -515,6 +515,27 @@ export default defineConfig({
 });
 ```
 
+## property: TestConfig.retryStrategy
+* since: v1.62
+- type: ?<[RetryStrategy]<"immediate"|"deferred">>
+
+Controls when failed tests are retried. Defaults to `'immediate'`.
+* `'immediate'` - A failed test is retried as soon as a worker is available, interleaved with the rest of the run. This is the default.
+* `'deferred'` - Retries are run only after all tests have had their first attempt, in parallel up to the configured number of [workers](#test-config-workers).
+
+Learn more about [test retries](../test-retries.md#retries).
+
+**Usage**
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  retries: 2,
+  retryStrategy: 'deferred',
+});
+```
+
 ## property: TestConfig.shard
 * since: v1.10
 - type: ?<[null]|[Object]>

packages/playwright/src/common/config.ts

@@ -48,6 +48,7 @@ export class FullConfigInternal {
   readonly projects: FullProjectInternal[] = [];
   readonly singleTSConfigPath?: string;
   readonly captureGitInfo: Config['captureGitInfo'];
+  readonly retryStrategy: 'immediate' | 'deferred';
   defineConfigWasUsed = false;
 
   globalSetups: string[] = [];
@@ -67,6 +68,7 @@ export class FullConfigInternal {
     this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p }));
     this.singleTSConfigPath = pathResolve(configDir, userConfig.tsconfig);
     this.captureGitInfo = userConfig.captureGitInfo;
+    this.retryStrategy = takeFirst(userConfig.retryStrategy, 'immediate');
 
     this.globalSetups = (Array.isArray(userConfig.globalSetup) ? userConfig.globalSetup : [userConfig.globalSetup]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined);
     this.globalTeardowns = (Array.isArray(userConfig.globalTeardown) ? userConfig.globalTeardown : [userConfig.globalTeardown]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined);

packages/playwright/src/common/configLoader.ts

@@ -243,6 +243,11 @@ function validateConfig(file: string, config: Config) {
       throw errorWithFile(file, `config.updateSnapshots must be one of "all", "changed", "missing" or "none"`);
   }
 
+  if ('retryStrategy' in config && config.retryStrategy !== undefined) {
+    if (typeof config.retryStrategy !== 'string' || !['immediate', 'deferred'].includes(config.retryStrategy))
+      throw errorWithFile(file, `config.retryStrategy must be one of "immediate" or "deferred"`);
+  }
+
   if ('tsconfig' in config && config.tsconfig !== undefined) {
     if (typeof config.tsconfig !== 'string')
       throw errorWithFile(file, `config.tsconfig must be a string`);

packages/playwright/src/runner/dispatcher.ts

@@ -152,7 +152,10 @@ export class Dispatcher {
 
     // 5. Possibly queue a new job with leftover tests and/or retries.
     if (!this._isStopped && result.newJob) {
-      this._queue.unshift(result.newJob);
+      if (this._testRun.config.retryStrategy === 'deferred')
+        this._queue.push(result.newJob);
+      else
+        this._queue.unshift(result.newJob);
       this._updateCounterForWorkerHash(result.newJob.workerHash, +1);
     }
   }

packages/playwright/types/test.d.ts

@@ -1640,6 +1640,30 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
    */
   retries?: number;
 
+  /**
+   * Controls when failed tests are retried. Defaults to `'immediate'`.
+   * - `'immediate'` - A failed test is retried as soon as a worker is available, interleaved with the rest of the
+   *   run. This is the default.
+   * - `'deferred'` - Retries are run only after all tests have had their first attempt, in parallel up to the
+   *   configured number of [workers](#test-config-workers).
+   *
+   * Learn more about [test retries](https://playwright.dev/docs/test-retries#retries).
+   *
+   * **Usage**
+   *
+   * ```js
+   * // playwright.config.ts
+   * import { defineConfig } from '@playwright/test';
+   *
+   * export default defineConfig({
+   *   retries: 2,
+   *   retryStrategy: 'deferred',
+   * });
+   * ```
+   *
+   */
+  retryStrategy?: "immediate"|"deferred";
+
   /**
    * Shard tests and execute only the selected shard. Specify in the one-based form like `{ total: 5, current: 2 }`.
    *

tests/playwright-test/retry.spec.ts

@@ -262,3 +262,39 @@ test('failed and skipped on retry should be marked as flaky', async ({ runInline
   expect(result.output).toContain('Failed on first run');
   expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry', location: expect.anything() }]);
 });
+
+test('should defer retries to the end of the run', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.js': `
+      module.exports = { retries: 3, retryStrategy: 'deferred' };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('a', ({}, testInfo) => {
+        console.log('\\n%%a-' + testInfo.retry);
+        expect(testInfo.retry).toBe(3);
+      });
+    `,
+    'b.test.js': `
+      import { test, expect } from '@playwright/test';
+      test.describe.configure({ retries: 2 });
+      test('b', ({}, testInfo) => {
+        console.log('\\n%%b-' + testInfo.retry);
+        expect(testInfo.retry).toBe(2);
+      });
+    `,
+  }, { workers: 1 });
+  expect(result.exitCode).toBe(0);
+  expect(result.flaky).toBe(2);
+  expect(result.results.length).toBe(7);
+  // First attempts run before any retry, and each retry round is interleaved.
+  expect(result.outputLines).toEqual([
+    'a-0',
+    'b-0',
+    'a-1',
+    'b-1',
+    'a-2',
+    'b-2',
+    'a-3',
+  ]);
+});