Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions DOCUMENTATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -357,6 +357,8 @@ testsprite test lint --steps ./refined.plan.json # the shape `test plan pu

Create a new test. Backend tests use `--code-file` (agents supply backend code directly); frontend tests use either `--code-file` or `--plan-from` (see [Plan file format](#plan-file-format)). With `--run --wait`, the CLI chains create → trigger → poll in a single invocation. `--step-timeout <ms>` sets a per-test step timeout from 1 to 60000 milliseconds on the code-file path. Backend tests can declare wave-ordering dependencies at create time — `--produces <var>` / `--needs <var>` (repeatable) and `--category <setup|main|teardown>` — and amend them later via `test update`.

Before creating a test, the CLI makes a best-effort check for an existing test with the same name. If that lookup fails, creation still proceeds; `--debug` reports the skipped advisory and its reason on stderr. Without `--debug`, lookup failures stay silent. This applies to both `--code-file` and `--plan-from`; the lookup is skipped under `--dry-run`.

`--plan-template` prints the canonical minimal plan-file skeleton to stdout and exits — pure-local, no network/credentials, ignores every other flag. The exact same example is embedded in `test create --help`.

```bash
Expand Down
95 changes: 95 additions & 0 deletions src/commands/test.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12094,3 +12094,98 @@ describe('runTestRun / runTestRerun — dashboard line on the queued-run output'
expect(bare.join('\n')).not.toContain('dashboard');
});
});

// #186: advisory failures stay non-fatal, but are diagnosable under --debug.
describe.each(['code', 'plan'] as const)('duplicate-name debug diagnostics (%s)', source => {
/** Run either create path with a controlled advisory response and capture both output streams. */
async function create({
debug = false,
verbose = false,
output = 'json',
lookupFails = true,
brokenSink = false,
}: {
debug?: boolean;
verbose?: boolean;
output?: 'json' | 'text';
lookupFails?: boolean;
brokenSink?: boolean;
} = {}) {
const { credentialsPath } = makeCreds();
const dir = mkdtempSync(join(tmpdir(), 'cli-dup-debug-'));
const codeFile = join(dir, 'test.py');
const planFrom = join(dir, 'plan.json');
const plan = {
projectId: 'project_alice',
type: 'frontend' as const,
name: 'Login',
planSteps: [{ type: 'assertion', description: 'Login succeeds' }],
};
writeFileSync(codeFile, '// test code');
writeFileSync(planFrom, JSON.stringify(plan));
const response = {
testId: 'test_new',
type: 'frontend',
codeVersion: 'v1',
createdAt: '2026-05-13T10:00:00.000Z',
};
let posts = 0;
let lookups = 0;
const stdout: string[] = [];
const stderr: string[] = [];
const deps = {
credentialsPath,
fetchImpl: makeFetch((_url, init) => {
if ((init.method ?? 'GET') === 'GET') {
lookups++;
return lookupFails
? {
status: 403,
body: { error: { code: 'AUTH_FORBIDDEN', message: 'Forbidden', requestId: 'r1' } },
}
: { body: { items: [] } };
}
posts++;
return { body: response };
}),
stdout: (line: string) => stdout.push(line),
stderr: (line: string) => {
stderr.push(line);
if (brokenSink && line.startsWith('[debug] duplicate-name')) throw new Error('sink failed');
},
};
const opts = { profile: 'default', output, debug, verbose, idempotencyKey: 'dup-debug-test' };
const result =
source === 'code'
? await runCreate({ ...opts, ...plan, codeFile }, deps)
: await runCreateFromPlan({ ...opts, planFrom }, deps);
expect(posts).toBe(1);
expect(lookups).toBe(1);
expect(result.testId).toBe(response.testId);
return { stdout: stdout.join('\n'), stderr: stderr.join('\n') };
}

it.each(['json', 'text'] as const)('keeps non-debug %s output byte-identical', async output => {
const baseline = await create({ output, lookupFails: false });
expect(await create({ output })).toEqual(baseline);
expect(await create({ output, verbose: true })).toEqual(baseline);
});

it.each(['json', 'text'] as const)(
'reports the swallowed reason only on stderr (%s)',
async output => {
const baseline = await create({ output });
const actual = await create({ output, debug: true });
expect(actual.stdout).toBe(baseline.stdout);
expect(actual.stderr).toContain('[debug] duplicate-name advisory skipped: Forbidden');
if (output === 'json') expect(JSON.parse(actual.stdout).testId).toBe('test_new');
const success = await create({ output, debug: true, lookupFails: false });
expect(success.stderr).not.toContain('duplicate-name advisory skipped');
},
);

it('still creates when diagnostic delivery throws', async () => {
const result = await create({ debug: true, brokenSink: true });
expect(JSON.parse(result.stdout).testId).toBe('test_new');
});
});
27 changes: 18 additions & 9 deletions src/commands/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1132,23 +1132,24 @@ function assertChainedRunKeyFits(
}
}

/** Short deadline for the advisory duplicate-name lookup (5 s). */
const DUP_NAME_ADVISORY_TIMEOUT_MS = 5_000;

/**
* B3 / Fix 4: best-effort duplicate-name advisory shared by `runCreate`
* and `runCreateFromPlan`. One-page lookup (pageSize=100) — not
* exhaustive but cheap. Silently swallows all errors; must never block
* or fail the caller's create.
* exhaustive but cheap. Reports swallowed errors only under --debug; must
* never block or fail the caller's create.
*
* Skip when `projectId` or `name` is absent (e.g. plan not yet parsed)
* or when the caller is in dry-run mode.
*/
/** Short deadline for the advisory duplicate-name lookup (5 s). */
const DUP_NAME_ADVISORY_TIMEOUT_MS = 5_000;

async function emitDupNameAdvisoryIfNeeded(
client: HttpClient,
projectId: string | undefined,
name: string | undefined,
stderrFn: (line: string) => void,
debug: boolean,
): Promise<void> {
if (!projectId || !name) return;
// B: the advisory lookup must NEVER block the create critical path.
Expand Down Expand Up @@ -1184,8 +1185,16 @@ async function emitDupNameAdvisoryIfNeeded(
`Use \`testsprite test update ${match.id}\` to modify it, or proceed to create a duplicate.`,
);
}
} catch {
// Swallow — this is best-effort; must not block the create.
} catch (error) {
// Diagnostics are best-effort too: a broken sink must not block creation.
if (debug) {
try {
const reason = error instanceof Error ? error.message : String(error);
stderrFn(`[debug] duplicate-name advisory skipped: ${reason}`);
} catch {
// Preserve the advisory's failure isolation if diagnostic delivery fails.
}
}
} finally {
clearTimeout(timer);
}
Expand Down Expand Up @@ -1381,7 +1390,7 @@ export async function runCreate(
// B3: best-effort duplicate-name advisory. Skip under --dry-run.
if (!opts.dryRun) {
const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`));
await emitDupNameAdvisoryIfNeeded(client, projectId, opts.name, stderrFn);
await emitDupNameAdvisoryIfNeeded(client, projectId, opts.name, stderrFn, opts.debug);
}

const response = await client.post<CliCreateTestResponse>('/tests', {
Expand Down Expand Up @@ -3058,7 +3067,7 @@ export async function runCreateFromPlan(
// The plan's projectId + name are available after validation above. Skip
// under dry-run (no network calls); swallow all errors (advisory only).
if (!opts.dryRun) {
await emitDupNameAdvisoryIfNeeded(client, plan.projectId, plan.name, stderrFn);
await emitDupNameAdvisoryIfNeeded(client, plan.projectId, plan.name, stderrFn, opts.debug);
}

const response = await client.post<CliCreateFromPlanResponse>('/tests', {
Expand Down
Loading