From 0937f3dfb5003e2c6848122f12a3ffb9e59ac5ef Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Tue, 18 Aug 2026 17:46:37 -0700 Subject: [PATCH 1/4] chore: scope AFS NUT deploy to its own AiTestingDefinition The agentforce-studio (AFS) NUT before-all deployed the whole aiTestingDefinitions/ directory via `sourcepath: [metaDir]`, which also swept in the static ReturnsCheckoutSuite fixture. That fixture targets a non-deployed multi-agent subject and exercises every scorer variant (incl. agent_handoff_match, valid only for multi-agent orchestrators), so the server rejected the deploy: Test case 4 ... topic is required and cannot be blank Test case 6: scorer 'agent_handoff_match' is only supported for multi-agent orchestrators ReturnsCheckoutSuite is an offline generate/compile fixture and is not deployed anywhere else. Scope the deploy to the file the NUT authored so the fixture keeps full scorer coverage for the generate NUTs without being sent to a server that has no multi-agent subject for it. --- test/nuts/z4.agent.test.AFS.nut.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/nuts/z4.agent.test.AFS.nut.ts b/test/nuts/z4.agent.test.AFS.nut.ts index 186f2f1d..00a69450 100644 --- a/test/nuts/z4.agent.test.AFS.nut.ts +++ b/test/nuts/z4.agent.test.AFS.nut.ts @@ -123,9 +123,11 @@ describe('agent test (agentforce-studio)', function () { writeFileSync(join(metaDir, `${afsTestName}.aiTestingDefinition-meta.xml`), metaXml, 'utf8'); console.log(`Wrote AiTestingDefinition metadata to ${metaDir}`); - // Deploy the definition + // Deploy the definition. Scope to just the file we authored — deploying the whole + // aiTestingDefinitions dir would also sweep in static fixtures (e.g. ReturnsCheckoutSuite, + // which targets a non-deployed multi-agent subject) and fail server-side validation. const cs = await ComponentSetBuilder.build({ - sourcepath: [metaDir], + sourcepath: [join(metaDir, `${afsTestName}.aiTestingDefinition-meta.xml`)], }); const deploy = await cs.deploy({ usernameOrConnection: getUsername() }); const deployResult = await deploy.pollStatus({ frequency: Duration.seconds(10), timeout: Duration.minutes(10) }); From 74ca7d650dad9be5bc811ac4a97d767c02907d35 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Tue, 18 Aug 2026 21:42:17 -0700 Subject: [PATCH 2/4] fix: report in-progress status when agent test run --wait times out On a client-side poll timeout, `agent test run --wait` removed the test cache entry and returned a hardcoded status of COMPLETED with no testCases. That both broke the `agent test resume --job-id` path it had just advised and masked a still-running (or slow) test as a passing, empty COMPLETED result. Treat a poll timeout as not-completed: leave the cache entry in place so resume works, skip result rendering when there are no results, and return the honest IN_PROGRESS status instead of COMPLETED. --- src/commands/agent/test/run.ts | 13 ++++++++++--- src/testStages.ts | 5 ++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 75c90444..262ed4d2 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -178,10 +178,18 @@ export default class AgentTestRun extends SfCommand { ); } - if (completed) await agentTestCache.removeCacheEntry(response.runId); - this.mso.stop(); + // A client-side poll timeout does not mean the run finished — it is still running + // server-side. Leave the cache entry in place so `agent test resume` works (poll() has + // already printed the resume hint) and report the in-progress status rather than masking + // the timeout as COMPLETED with no results. + if (!completed || !detailsResponse) { + return { status: 'IN_PROGRESS', runId: response.runId }; + } + + await agentTestCache.removeCacheEntry(response.runId); + await handleTestResults({ id: response.runId, format: flags['result-format'], @@ -195,7 +203,6 @@ export default class AgentTestRun extends SfCommand { // Test assertion failures are business logic and should not affect exit code // Only applicable to legacy responses (Agentforce Studio doesn't have test case status) if ( - detailsResponse && 'subjectName' in detailsResponse && detailsResponse.testCases.some((tc) => 'status' in tc && tc.status === 'ERROR') ) { diff --git a/src/testStages.ts b/src/testStages.ts index 958e37cb..ba1311f2 100644 --- a/src/testStages.ts +++ b/src/testStages.ts @@ -113,7 +113,10 @@ export class TestStages { this.stop('async'); this.ux.log(`Client timed out after ${wait.minutes} minutes.`); this.ux.log(`Run ${colorize('dim', `sf agent test resume --job-id ${id}`)} to resuming watching this test.`); - return { completed: true }; + // The client stopped watching, but the run is still going server-side — it did NOT + // complete. Report completed:false so the caller preserves the cache entry (for + // `agent test resume`) and does not mask the timeout as a COMPLETED result. + return { completed: false }; } else { this.error(); throw e; From a529955d2003d0327b06463ae61d70ce231bc379 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Wed, 19 Aug 2026 08:09:30 -0700 Subject: [PATCH 3/4] chore: log per-attempt publish timing in z2 NUT to surface stalls Adds a beforeEach/afterEach timing hook to the publish NUTs that logs each attempt's wall-clock and pass/fail state. Because it fires on every attempt, it cuts through this.retries() (which reports only the final passing attempt and hides a slow first one). Paired with the bounded publish-poll timeout in @salesforce/agents, a genuine stall also identifies which poll hung via its error. Clean test-code logging with no CI env-var/log-level machinery; safe to keep merged. --- test/nuts/z2.agent.publish.nut.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/nuts/z2.agent.publish.nut.ts b/test/nuts/z2.agent.publish.nut.ts index 1f11aef8..79768371 100644 --- a/test/nuts/z2.agent.publish.nut.ts +++ b/test/nuts/z2.agent.publish.nut.ts @@ -87,6 +87,25 @@ describe('agent publish authoring-bundle NUTs', function () { connection = org.getConnection(); }); + // Per-attempt timing so a slow publish is visible in CI. This fires on EVERY attempt + // (including the ones mocha's this.retries() would otherwise hide by reporting only the + // final pass), so a stalled first attempt shows up here as a large elapsed with a failed + // state. Paired with the bounded publish-poll timeout, a genuine stall also surfaces which + // poll hung via its error (agentRetrievalError vs authoringBundleDeploymentError). + let attemptStart = 0; + beforeEach(() => { + attemptStart = Date.now(); + }); + afterEach(function () { + const elapsedSec = Math.round((Date.now() - attemptStart) / 1000); + // eslint-disable-next-line no-console + console.log( + `[z2 publish timing] "${this.currentTest?.title ?? 'unknown'}" attempt took ${elapsedSec}s (${ + this.currentTest?.state ?? 'unknown' + })` + ); + }); + it('should publish a new agent (first version)', async function () { // Increase timeout to 30 minutes since deployment can take a long time this.timeout(30 * 60 * 1000); // 30 minutes From c85871423e44471666e5a1cbf1f6b3daf543c037 Mon Sep 17 00:00:00 2001 From: Nico Pappagianis Date: Wed, 19 Aug 2026 10:32:25 -0700 Subject: [PATCH 4/4] fix: report IN_PROGRESS on resume timeout and make AFS NUT tolerant @W-23915028@ resume.ts had the same masking bug as run.ts: on a client-side poll timeout it called handleTestResults with an undefined response and returned a hardcoded COMPLETED. Guard the timeout path to return IN_PROGRESS and leave the cache entry in place so the run can be resumed again. The z4 AFS NUT was green only because of that lie. Make it robust to the honest behavior: accept COMPLETED or IN_PROGRESS on the --wait run, use the run's id regardless of completion, and branch the resume test on whether the primary run finished (fresh async run + resume by job-id when completed; resume --use-most-recent against the in-flight run otherwise, avoiding the AFS one-run-per-definition collision). --- src/commands/agent/test/resume.ts | 21 ++++--- test/nuts/z4.agent.test.AFS.nut.ts | 91 +++++++++++++++++++----------- 2 files changed, 69 insertions(+), 43 deletions(-) diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index 8cdce823..0697cf1f 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -141,10 +141,18 @@ export default class AgentTestResume extends SfCommand { ); } - if (completed) await agentTestCache.removeCacheEntry(runId); - this.mso.stop(); + // A client-side poll timeout does not mean the run finished — it is still running + // server-side. Leave the cache entry in place so the run can be resumed again (poll() has + // already printed the resume hint) and report the in-progress status rather than masking the + // timeout as COMPLETED with no results. + if (!completed || !response) { + return { status: 'IN_PROGRESS', runId }; + } + + await agentTestCache.removeCacheEntry(runId); + await handleTestResults({ id: runId, format: resultFormat ?? flags['result-format'], @@ -157,16 +165,11 @@ export default class AgentTestResume extends SfCommand { // Set exit code to 1 only for execution errors (tests couldn't run properly) // Test assertion failures are business logic and should not affect exit code // Only applicable to legacy responses (Agentforce Studio doesn't have test case status) - if ( - response && - 'subjectName' in response && - response.testCases.some((tc) => 'status' in tc && tc.status === 'ERROR') - ) { + if ('subjectName' in response && response.testCases.some((tc) => 'status' in tc && tc.status === 'ERROR')) { process.exitCode = 1; } - // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion - return { ...response!, runId, status: 'COMPLETED' } as AgentTestRunResult; + return { ...response, runId, status: 'COMPLETED' } as AgentTestRunResult; } protected catch(error: Error | SfError | CLIError): Promise { diff --git a/test/nuts/z4.agent.test.AFS.nut.ts b/test/nuts/z4.agent.test.AFS.nut.ts index 00a69450..13eede17 100644 --- a/test/nuts/z4.agent.test.AFS.nut.ts +++ b/test/nuts/z4.agent.test.AFS.nut.ts @@ -141,8 +141,10 @@ describe('agent test (agentforce-studio)', function () { console.log(`Deployed AiTestingDefinition '${afsTestName}'`); }); - // Set by the run test, consumed by the results tests (Mocha runs describes sequentially) - let completedRunId: string; + // Set by the run test, consumed by the results/resume tests (Mocha runs describes sequentially). + // The AFS eval can outlast the client --wait window, so the run may still be IN_PROGRESS here. + let sharedRunId: string; + let primaryRunCompleted = false; describe('agent test list', () => { it('should include the AFS test definition in list', async () => { @@ -164,20 +166,26 @@ describe('agent test (agentforce-studio)', function () { { ensureExitCode: 0 } ).jsonOutput; - expect(output?.result.status).to.equal('COMPLETED'); - expect(output?.result.runId.startsWith('3A2')).to.be.true; const result = output?.result as AgentTestRunResult & { testCases?: unknown[] }; - expect(result?.testCases).to.be.an('array'); + // The AFS eval can take longer than the --wait window. A client-side timeout is a valid + // outcome that now reports IN_PROGRESS (rather than masking as COMPLETED), so accept either + // and assert only the run shape. testCases is only guaranteed once the run has COMPLETED. + expect(result?.status).to.be.oneOf(['COMPLETED', 'IN_PROGRESS']); + expect(result?.runId.startsWith('3A2')).to.be.true; expect(result).to.not.have.property('subjectName'); + if (result?.status === 'COMPLETED') { + expect(result?.testCases).to.be.an('array'); + primaryRunCompleted = true; + } - completedRunId = output!.result.runId; + sharedRunId = output!.result.runId; }); }); describe('agent test results', () => { it('should fetch AFS results by job ID (json)', async () => { const output = execCmd( - `agent test results --job-id ${completedRunId} --target-org ${getUsername()} --json`, + `agent test results --job-id ${sharedRunId} --target-org ${getUsername()} --json`, { ensureExitCode: 0 } ).jsonOutput; @@ -189,7 +197,7 @@ describe('agent test (agentforce-studio)', function () { it('should support human result format', () => { const output = execCmd( - `agent test results --job-id ${completedRunId} --result-format human --target-org ${getUsername()}`, + `agent test results --job-id ${sharedRunId} --result-format human --target-org ${getUsername()}`, { ensureExitCode: 0 } ); expect(output.shellOutput.stdout).to.be.a('string').with.length.greaterThan(0); @@ -197,7 +205,7 @@ describe('agent test (agentforce-studio)', function () { it('should support junit result format', () => { const output = execCmd( - `agent test results --job-id ${completedRunId} --result-format junit --target-org ${getUsername()}`, + `agent test results --job-id ${sharedRunId} --result-format junit --target-org ${getUsername()}`, { ensureExitCode: 0 } ); expect(output.shellOutput.stdout).to.include(' { const output = execCmd( - `agent test results --job-id ${completedRunId} --result-format tap --target-org ${getUsername()}`, + `agent test results --job-id ${sharedRunId} --result-format tap --target-org ${getUsername()}`, { ensureExitCode: 0 } ); expect(output.shellOutput.stdout).to.include('TAP version 13'); @@ -214,36 +222,51 @@ describe('agent test (agentforce-studio)', function () { }); describe('agent test resume', () => { - it('should start async then resume by job ID, and support --use-most-recent', async () => { - // Clear any stale entries before the run - const cacheBefore = await AgentTestCache.create(); - cacheBefore.clear(); - await cacheBefore.write(); + it('should resume the in-flight run (or a fresh async run) and honor the cache', async function () { + this.timeout(30 * 60 * 1000); - // One async start covers both resume paths - const runResult = execCmd( - `agent test run --api-name ${afsTestName} --target-org ${getUsername()} --json`, - { ensureExitCode: 0 } - ).jsonOutput; + if (primaryRunCompleted) { + // The primary --wait run already finished, so its cache entry was removed and no run is + // in flight. Start a fresh async run to exercise NEW + cache-write, then resume by job-id. + const cacheBefore = await AgentTestCache.create(); + cacheBefore.clear(); + await cacheBefore.write(); - expect(runResult?.result.runId.startsWith('3A2')).to.be.true; - expect(runResult?.result.status).to.equal('NEW'); + const runResult = execCmd( + `agent test run --api-name ${afsTestName} --target-org ${getUsername()} --json`, + { ensureExitCode: 0 } + ).jsonOutput; - // Re-read from disk — the run command wrote the cache entry in a subprocess - const cache = await AgentTestCache.create(); - expect(cache.resolveFromCache().runnerType).to.equal('agentforce-studio'); + expect(runResult?.result.runId.startsWith('3A2')).to.be.true; + expect(runResult?.result.status).to.equal('NEW'); - const output = execCmd( - `agent test resume --job-id ${runResult?.result.runId} --target-org ${getUsername()} --json`, - { ensureExitCode: 0 } - ).jsonOutput; + // Re-read from disk — the run command wrote the cache entry in a subprocess + const cache = await AgentTestCache.create(); + expect(cache.resolveFromCache().runnerType).to.equal('agentforce-studio'); + + const output = execCmd( + `agent test resume --job-id ${runResult?.result.runId} --target-org ${getUsername()} --json`, + { ensureExitCode: 0 } + ).jsonOutput; + + // Resume may itself time out (the eval can outlast the poll window); both are valid. + expect(output?.result.status).to.be.oneOf(['COMPLETED', 'IN_PROGRESS']); + expect(output?.result.runId.startsWith('3A2')).to.be.true; + } else { + // The primary --wait run timed out client-side and is still running server-side. Our fix + // preserves the cache entry on a timeout, so resume --use-most-recent picks it up. Starting + // a NEW run here would collide — AFS allows only one run per definition at a time. + const cache = await AgentTestCache.create(); + expect(cache.resolveFromCache().runnerType).to.equal('agentforce-studio'); - expect(output?.result.status).to.equal('COMPLETED'); - expect(output?.result.runId.startsWith('3A2')).to.be.true; + const output = execCmd( + `agent test resume --use-most-recent --target-org ${getUsername()} --json`, + { ensureExitCode: 0 } + ).jsonOutput; - // Re-read from disk — resume removes the entry in a subprocess - const cacheAfter = await AgentTestCache.create(); - expect(() => cacheAfter.resolveFromCache()).to.throw('Could not find a runId to resume'); + expect(output?.result.status).to.be.oneOf(['COMPLETED', 'IN_PROGRESS']); + expect(output?.result.runId).to.equal(sharedRunId); + } }); });