feat: retain-on-failure-and-retries trace option

docs/src/test-api/class-testoptions.md

@@ -588,8 +588,8 @@ export default defineConfig({
 
 ## property: TestOptions.trace
 * since: v1.10
-- type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"retain-on-first-failure">>
-  - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure">> Trace recording mode.
+- type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"retain-on-first-failure"|"retain-on-failure-and-retries">>
+  - `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure"|"retain-on-failure-and-retries">> Trace recording mode.
   - `attachments` ?<[boolean]> Whether to include test attachments. Defaults to true. Optional.
   - `screenshots` ?<[boolean]> Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview. Defaults to true. Optional.
   - `snapshots` ?<[boolean]> Whether to capture DOM snapshot on every action. Defaults to true. Optional.
@@ -602,6 +602,7 @@ Whether to record trace for each test. Defaults to `'off'`.
 * `'on-all-retries'`: Record trace only when retrying a test.
 * `'retain-on-failure'`: Record trace for each test. When test run passes, remove the recorded trace.
 * `'retain-on-first-failure'`: Record trace for the first run of each test, but not for retries. When test run passes, remove the recorded trace.
+* `'retain-on-failure-and-retries'`: Record trace for each test run. Retains all traces when an attempt fails.
 
 For more control, pass an object that specifies `mode` and trace features to enable.
 

docs/src/test-cli-js.md

@@ -106,7 +106,7 @@ npx playwright test --ui
 | `--test-list <file>` | Path to a file containing a list of tests to run. See [test list](#test-list) for details. |
 | `--test-list-invert <file>` | Path to a file containing a list of tests to skip. See [test list](#test-list) for details.  |
 | `--timeout <timeout>` | Specify test timeout threshold in milliseconds, zero for unlimited (default: 30 seconds). |
-| `--trace <mode>` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure`, `retain-on-first-failure`. |
+| `--trace <mode>` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure`, `retain-on-first-failure`, `retain-on-failure-and-retries`. |
 | `--tsconfig <path>` | Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately). |
 | `--ui` | Run tests in interactive UI mode. |
 | `--ui-host <host>` | Host to serve UI on; specifying this option opens UI in a browser tab. |

packages/playwright/src/program.ts

@@ -403,7 +403,7 @@ function resolveReporter(id: string) {
   return require.resolve(id, { paths: [process.cwd()] });
 }
 
-const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure', 'retain-on-first-failure'];
+const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure', 'retain-on-first-failure', 'retain-on-failure-and-retries'];
 
 // Note: update docs/src/test-cli-js.md when you update this, program is the source of truth.
 

packages/playwright/src/worker/testTracing.ts

@@ -83,6 +83,9 @@ export class TestTracing {
     if (this._options?.mode === 'retain-on-first-failure' && this._testInfo.retry === 0)
       return true;
 
+    if (this._options?.mode === 'retain-on-failure-and-retries')
+      return true;
+
     return false;
   }
 
@@ -159,6 +162,8 @@ export class TestTracing {
     if (!this._options)
       return true;
     const testFailed = this._testInfo.status !== this._testInfo.expectedStatus;
+    if (this._options.mode === 'retain-on-failure-and-retries')
+      return !testFailed && this._testInfo.retry === 0;
     return !testFailed && (this._options.mode === 'retain-on-failure' || this._options.mode === 'retain-on-first-failure');
   }
 

packages/playwright/types/test.d.ts

@@ -6903,6 +6903,7 @@ export interface PlaywrightWorkerOptions {
    * - `'retain-on-failure'`: Record trace for each test. When test run passes, remove the recorded trace.
    * - `'retain-on-first-failure'`: Record trace for the first run of each test, but not for retries. When test run
    *   passes, remove the recorded trace.
+   * - `'retain-on-failure-and-retries'`: Record trace for each test run. Retains all traces when an attempt fails.
    *
    * For more control, pass an object that specifies `mode` and trace features to enable.
    *
@@ -6953,7 +6954,7 @@ export interface PlaywrightWorkerOptions {
 }
 
 export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure';
-export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure';
+export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries';
 export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
 export type AgentOptions = {
   provider?: {

tests/playwright-test/playwright.artifacts.spec.ts

@@ -426,6 +426,52 @@ test('should work with trace: retain-on-first-failure', async ({ runInlineTest }
   ]);
 });
 
+test('should work with trace: retain-on-failure-and-retries', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    ...testFiles,
+    'playwright.config.ts': `
+      module.exports = { use: { trace: 'retain-on-failure-and-retries' } };
+    `,
+  }, { workers: 1, retries: 2 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(5);
+  expect(result.failed).toBe(5);
+  expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
+    '.last-run.json',
+    'artifacts-failing',
+    '  trace.zip',
+    'artifacts-failing-retry1',
+    '  trace.zip',
+    'artifacts-failing-retry2',
+    '  trace.zip',
+    'artifacts-own-context-failing',
+    '  trace.zip',
+    'artifacts-own-context-failing-retry1',
+    '  trace.zip',
+    'artifacts-own-context-failing-retry2',
+    '  trace.zip',
+    'artifacts-persistent-failing',
+    '  trace.zip',
+    'artifacts-persistent-failing-retry1',
+    '  trace.zip',
+    'artifacts-persistent-failing-retry2',
+    '  trace.zip',
+    'artifacts-shared-shared-failing',
+    '  trace.zip',
+    'artifacts-shared-shared-failing-retry1',
+    '  trace.zip',
+    'artifacts-shared-shared-failing-retry2',
+    '  trace.zip',
+    'artifacts-two-contexts-failing',
+    '  trace.zip',
+    'artifacts-two-contexts-failing-retry1',
+    '  trace.zip',
+    'artifacts-two-contexts-failing-retry2',
+    '  trace.zip',
+  ]);
+});
+
 test('should take screenshot when page is closed in afterEach', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
     'playwright.config.ts': `

tests/playwright-test/playwright.trace.spec.ts

@@ -385,7 +385,7 @@ test('should respect --trace', async ({ runInlineTest }, testInfo) => {
   expect(fs.existsSync(testInfo.outputPath('test-results', 'a-test-1', 'trace.zip'))).toBeTruthy();
 });
 
-for (const mode of ['off', 'retain-on-failure', 'on-first-retry', 'on-all-retries', 'retain-on-first-failure']) {
+for (const mode of ['off', 'retain-on-failure', 'on-first-retry', 'on-all-retries', 'retain-on-first-failure', 'retain-on-failure-and-retries']) {
   test(`trace:${mode} should not create trace zip artifact if page test passed`, async ({ runInlineTest }) => {
     const result = await runInlineTest({
       'a.spec.ts': `
@@ -1152,6 +1152,46 @@ test('trace:retain-on-first-failure should create trace if request context is di
   expect(result.failed).toBe(1);
 });
 
+test('trace:retain-on-failure-and-retries should keep all traces when test is flaky', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('flaky', async ({ page }) => {
+        await page.goto('about:blank');
+        expect(test.info().retry).toBe(1);
+      });
+    `,
+  }, { trace: 'retain-on-failure-and-retries', retries: 1 });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.flaky).toBe(1);
+
+  const firstRunTracePath = testInfo.outputPath('test-results', 'a-flaky', 'trace.zip');
+  expect(fs.existsSync(firstRunTracePath)).toBeTruthy();
+  const retryTracePath = testInfo.outputPath('test-results', 'a-flaky-retry1', 'trace.zip');
+  expect(fs.existsSync(retryTracePath)).toBeTruthy();
+});
+
+test('trace:retain-on-failure-and-retries should keep all traces when test fails on retries', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('fail', async ({ page }) => {
+        await page.goto('about:blank');
+        expect(true).toBe(false);
+      });
+    `,
+  }, { trace: 'retain-on-failure-and-retries', retries: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+
+  const firstRunTracePath = testInfo.outputPath('test-results', 'a-fail', 'trace.zip');
+  expect(fs.existsSync(firstRunTracePath)).toBeTruthy();
+  const retryTracePath = testInfo.outputPath('test-results', 'a-fail-retry1', 'trace.zip');
+  expect(fs.existsSync(retryTracePath)).toBeTruthy();
+});
+
 test('should not corrupt actions when no library trace is present', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'a.spec.ts': `

utils/generate_types/overrides-test.d.ts

@@ -263,7 +263,7 @@ export interface PlaywrightWorkerOptions {
 }
 
 export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure';
-export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure';
+export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure' | 'retain-on-failure-and-retries';
 export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
 export type AgentOptions = {
   provider?: {