From 442430b1f42c9364ec881ce21bd1c67df0b51057 Mon Sep 17 00:00:00 2001 From: yuhao Date: Fri, 25 Sep 2026 22:51:28 +0800 Subject: [PATCH 1/3] fix(test): diagnose skipped duplicate-name checks in debug mode --- DOCUMENTATION.md | 2 + src/commands/test.test.ts | 94 +++++++++++++++++++++++++++++++++++++++ src/commands/test.ts | 21 ++++++--- 3 files changed, 111 insertions(+), 6 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index fd8a654..4314776 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -357,6 +357,8 @@ testsprite test lint --steps ./refined.plan.json # the shape `test plan pu Create a new test. Backend tests use `--code-file` (agents supply backend code directly); frontend tests use either `--code-file` or `--plan-from` (see [Plan file format](#plan-file-format)). With `--run --wait`, the CLI chains create → trigger → poll in a single invocation. `--step-timeout ` sets a per-test step timeout from 1 to 60000 milliseconds on the code-file path. Backend tests can declare wave-ordering dependencies at create time — `--produces ` / `--needs ` (repeatable) and `--category ` — and amend them later via `test update`. +Before creating a test, the CLI makes a best-effort check for an existing test with the same name. If that lookup fails, creation still proceeds; `--debug` reports the skipped advisory and its reason on stderr. Without `--debug`, lookup failures stay silent. This applies to both `--code-file` and `--plan-from`; the lookup is skipped under `--dry-run`. + `--plan-template` prints the canonical minimal plan-file skeleton to stdout and exits — pure-local, no network/credentials, ignores every other flag. The exact same example is embedded in `test create --help`. ```bash diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index a549c12..03f7332 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -12094,3 +12094,97 @@ describe('runTestRun / runTestRerun — dashboard line on the queued-run output' expect(bare.join('\n')).not.toContain('dashboard'); }); }); + +// #186: advisory failures stay non-fatal, but are diagnosable under --debug. +describe.each(['code', 'plan'] as const)('duplicate-name debug diagnostics (%s)', source => { + async function create({ + debug = false, + verbose = false, + output = 'json', + lookupFails = true, + brokenSink = false, + }: { + debug?: boolean; + verbose?: boolean; + output?: 'json' | 'text'; + lookupFails?: boolean; + brokenSink?: boolean; + } = {}) { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-dup-debug-')); + const codeFile = join(dir, 'test.py'); + const planFrom = join(dir, 'plan.json'); + const plan = { + projectId: 'project_alice', + type: 'frontend' as const, + name: 'Login', + planSteps: [{ type: 'assertion', description: 'Login succeeds' }], + }; + writeFileSync(codeFile, '// test code'); + writeFileSync(planFrom, JSON.stringify(plan)); + const response = { + testId: 'test_new', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-05-13T10:00:00.000Z', + }; + let posts = 0; + let lookups = 0; + const stdout: string[] = []; + const stderr: string[] = []; + const deps = { + credentialsPath, + fetchImpl: makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') { + lookups++; + return lookupFails + ? { + status: 403, + body: { error: { code: 'AUTH_FORBIDDEN', message: 'Forbidden', requestId: 'r1' } }, + } + : { body: { items: [] } }; + } + posts++; + return { body: response }; + }), + stdout: (line: string) => stdout.push(line), + stderr: (line: string) => { + stderr.push(line); + if (brokenSink && line.startsWith('[debug] duplicate-name')) throw new Error('sink failed'); + }, + }; + const opts = { profile: 'default', output, debug, verbose, idempotencyKey: 'dup-debug-test' }; + const result = + source === 'code' + ? await runCreate({ ...opts, ...plan, codeFile }, deps) + : await runCreateFromPlan({ ...opts, planFrom }, deps); + expect(posts).toBe(1); + expect(lookups).toBe(1); + expect(result.testId).toBe(response.testId); + return { stdout: stdout.join('\n'), stderr: stderr.join('\n') }; + } + + it.each(['json', 'text'] as const)('keeps non-debug %s output byte-identical', async output => { + const baseline = await create({ output, lookupFails: false }); + expect(await create({ output })).toEqual(baseline); + expect(await create({ output, verbose: true })).toEqual(baseline); + }); + + it.each(['json', 'text'] as const)( + 'reports the swallowed reason only on stderr (%s)', + async output => { + const baseline = await create({ output }); + const actual = await create({ output, debug: true }); + expect(actual.stdout).toBe(baseline.stdout); + expect(actual.stderr).toContain('[debug] duplicate-name advisory skipped: Forbidden'); + if (output === 'json') expect(JSON.parse(actual.stdout).testId).toBe('test_new'); + const success = await create({ output, debug: true, lookupFails: false }); + expect(success.stderr).not.toContain('duplicate-name advisory skipped'); + }, + ); + + it('still creates when diagnostic delivery throws', async () => { + const result = await create({ debug: true, brokenSink: true }); + expect(JSON.parse(result.stdout).testId).toBe('test_new'); + }); +}); diff --git a/src/commands/test.ts b/src/commands/test.ts index 53de3fa..3a67b46 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -1135,8 +1135,8 @@ function assertChainedRunKeyFits( /** * B3 / Fix 4: best-effort duplicate-name advisory shared by `runCreate` * and `runCreateFromPlan`. One-page lookup (pageSize=100) — not - * exhaustive but cheap. Silently swallows all errors; must never block - * or fail the caller's create. + * exhaustive but cheap. Reports swallowed errors only under --debug; must + * never block or fail the caller's create. * * Skip when `projectId` or `name` is absent (e.g. plan not yet parsed) * or when the caller is in dry-run mode. @@ -1149,6 +1149,7 @@ async function emitDupNameAdvisoryIfNeeded( projectId: string | undefined, name: string | undefined, stderrFn: (line: string) => void, + debug: boolean, ): Promise { if (!projectId || !name) return; // B: the advisory lookup must NEVER block the create critical path. @@ -1184,8 +1185,16 @@ async function emitDupNameAdvisoryIfNeeded( `Use \`testsprite test update ${match.id}\` to modify it, or proceed to create a duplicate.`, ); } - } catch { - // Swallow — this is best-effort; must not block the create. + } catch (error) { + // Diagnostics are best-effort too: a broken sink must not block creation. + if (debug) { + try { + const reason = error instanceof Error ? error.message : String(error); + stderrFn(`[debug] duplicate-name advisory skipped: ${reason}`); + } catch { + // Preserve the advisory's failure isolation if diagnostic delivery fails. + } + } } finally { clearTimeout(timer); } @@ -1381,7 +1390,7 @@ export async function runCreate( // B3: best-effort duplicate-name advisory. Skip under --dry-run. if (!opts.dryRun) { const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); - await emitDupNameAdvisoryIfNeeded(client, projectId, opts.name, stderrFn); + await emitDupNameAdvisoryIfNeeded(client, projectId, opts.name, stderrFn, opts.debug); } const response = await client.post('/tests', { @@ -3058,7 +3067,7 @@ export async function runCreateFromPlan( // The plan's projectId + name are available after validation above. Skip // under dry-run (no network calls); swallow all errors (advisory only). if (!opts.dryRun) { - await emitDupNameAdvisoryIfNeeded(client, plan.projectId, plan.name, stderrFn); + await emitDupNameAdvisoryIfNeeded(client, plan.projectId, plan.name, stderrFn, opts.debug); } const response = await client.post('/tests', { From 1eba87972f7e25594e44cc0127f4f243fce24614 Mon Sep 17 00:00:00 2001 From: yuhao Date: Fri, 25 Sep 2026 23:00:34 +0800 Subject: [PATCH 2/3] docs(test): attach advisory documentation to its function --- src/commands/test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands/test.ts b/src/commands/test.ts index 3a67b46..28a0e2a 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -1132,6 +1132,9 @@ function assertChainedRunKeyFits( } } +/** Short deadline for the advisory duplicate-name lookup (5 s). */ +const DUP_NAME_ADVISORY_TIMEOUT_MS = 5_000; + /** * B3 / Fix 4: best-effort duplicate-name advisory shared by `runCreate` * and `runCreateFromPlan`. One-page lookup (pageSize=100) — not @@ -1141,9 +1144,6 @@ function assertChainedRunKeyFits( * Skip when `projectId` or `name` is absent (e.g. plan not yet parsed) * or when the caller is in dry-run mode. */ -/** Short deadline for the advisory duplicate-name lookup (5 s). */ -const DUP_NAME_ADVISORY_TIMEOUT_MS = 5_000; - async function emitDupNameAdvisoryIfNeeded( client: HttpClient, projectId: string | undefined, From f826fff815631dd13d72eacf20118d5e238dd8d9 Mon Sep 17 00:00:00 2001 From: yuhao Date: Fri, 25 Sep 2026 23:05:19 +0800 Subject: [PATCH 3/3] docs(test): describe diagnostic regression fixture --- src/commands/test.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 03f7332..01d12a1 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -12097,6 +12097,7 @@ describe('runTestRun / runTestRerun — dashboard line on the queued-run output' // #186: advisory failures stay non-fatal, but are diagnosable under --debug. describe.each(['code', 'plan'] as const)('duplicate-name debug diagnostics (%s)', source => { + /** Run either create path with a controlled advisory response and capture both output streams. */ async function create({ debug = false, verbose = false,