diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 05ff0d5..0a5b168 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,13 +52,23 @@ jobs: echo "publish=true" >>"$GITHUB_OUTPUT" echo "::notice::Will publish ${{ steps.meta.outputs.name }}@${{ steps.meta.outputs.version }}." fi + # Preferred: npm Trusted Publishing (OIDC), no secret needed. Fallback: + # the NPM_TOKEN secret — npm only reads a token from .npmrc (a + # NODE_AUTH_TOKEN env var alone is ignored), so write .npmrc explicitly, + # and only when the secret is set (an empty _authToken breaks OIDC). - if: steps.check.outputs.publish == 'true' run: | set -euo pipefail + if [ -n "${NPM_TOKEN:-}" ]; then + echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >>"$HOME/.npmrc" + echo "::notice::Publishing with the NPM_TOKEN secret (fallback)." + else + echo "::notice::No NPM_TOKEN secret; relying on npm Trusted Publishing (OIDC)." + fi npx -y npm@11.5.1 --version npx -y npm@11.5.1 publish --provenance env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - if: steps.check.outputs.publish == 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 5a4c266..d190fb3 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,7 @@ Use `getHeaders` on a server config to inject fresh credentials at connect time, | `maxIterations` | Cap on **executed steps** across the run (every step counts, including those run after a `revise`). | | `maxStepsPerTask` | Cap on LLM steps inside a single executor call (multi-step tool calling). | | `maxRevisions` | Cap on `revise` decisions the replanner can make per run. Default `2`. | +| `replanAfter` | Replan trigger: `'failure'` (default; blocked step or a tool failure that stayed failed) \| `'always'` \| `(stepResult) => boolean \| Promise` (bounded by `llmTimeoutMs`; falls back to `'failure'` on error). | | `maxTotalTokens` | Soft cap on cumulative input + output tokens; checked between steps and triggers an early jump to synthesis when crossed. | | `llmTimeoutMs` / `llmMaxRetries` | Per-LLM-call timeout and retry budget. | | `toolSelectionStrategy` | `'all'` (default) gives the executor every tool each step; `'plan-narrowed'` exposes only `step.suggestedTools`. | diff --git a/package-lock.json b/package-lock.json index 3d2aa82..6b17f75 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dudko.dev/agent", - "version": "0.0.16", + "version": "0.0.17", "lockfileVersion": 7, "requires": true, "packages": { "": { "name": "@dudko.dev/agent", - "version": "0.0.16", + "version": "0.0.17", "funding": [ { "type": "individual", diff --git a/package.json b/package.json index 3b1a699..94310c1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dudko.dev/agent", - "version": "0.0.16", + "version": "0.0.17", "type": "module", "description": "Tool-using planning agent over MCP servers, built on the Vercel AI SDK.", "keywords": [ diff --git a/src/index.ts b/src/index.ts index 87ceb73..b7a0590 100644 --- a/src/index.ts +++ b/src/index.ts @@ -36,6 +36,7 @@ export type { LogLevel, ProviderType, ReplanCause, + ReplanTrigger, ToolSelectionStrategy, } from './types.ts' diff --git a/src/runner.ts b/src/runner.ts index 0c1872d..d4e0f46 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -17,24 +17,102 @@ import type { IRunSnapshot, IStepResult, IUsage, + ReplanTrigger, } from './types.ts' import { ATTR, withSpan } from './tracing.ts' import { combineSignals } from './utils.ts' -// Replanner is invoked when: +// The default ('failure') replan trigger fires when: // - the executor explicitly signalled a blocker (via the [BLOCKER] sentinel, // decoded into result.blocked); or -// - any tool call in this step failed. +// - a tool call in this step failed and STAYED failed. A failure that a +// later call to the same tool retried successfully is self-corrected - +// the executor's multi-step loop already recovered, so it must not force +// a replan. // Both signals are language-independent and structural, so we don't parse the // summary's prose. shouldCallReplanner is exported for direct unit testing. export const shouldCallReplanner = (result: IStepResult): boolean => { if (result.blocked) { return true } - if (result.toolCalls.some((c) => !c.ok)) { + const { toolCalls } = result + return toolCalls.some( + (c, i) => !c.ok && !toolCalls.slice(i + 1).some((later) => later.ok && later.name === c.name), + ) +} + +export interface IReplanTriggerOptions { + signal?: AbortSignal + timeoutMs?: number + // Called with whatever a host predicate threw before falling back to 'failure'. + onError?: (err: unknown) => void +} + +// Resolve the configured `replanAfter` trigger for one step result. 'failure' +// is the classic shouldCallReplanner; 'always' consults the replanner after +// every step; a host predicate decides per result. A predicate that throws, +// rejects, or outlives the watchdog/abort falls back to 'failure' behaviour so +// a buggy or hung predicate can never stall the run. +export const replanTriggered = async ( + trigger: ReplanTrigger | undefined, + result: IStepResult, + opts: IReplanTriggerOptions = {}, +): Promise => { + if (trigger === 'always') { return true } - return false + if (typeof trigger !== 'function') { + return shouldCallReplanner(result) + } + const fallback = (): boolean => shouldCallReplanner(result) + const { signal } = opts + const timeoutMs = opts.timeoutMs ?? 0 + if (signal?.aborted) { + return fallback() + } + // Never rejects: a sync throw or async rejection resolves to the fallback. + const decided = (async (): Promise => { + try { + return Boolean(await trigger(result)) + } catch (err) { + opts.onError?.(err) + return fallback() + } + })() + const hasTimeout = Number.isFinite(timeoutMs) && timeoutMs > 0 + if (!hasTimeout && !signal) { + // No watchdog configured - just await the predicate. + return decided + } + // Watchdog: fall back when the timeout elapses or the run aborts. We use a + // ref'd setTimeout rather than AbortSignal.timeout on purpose - an + // AbortSignal.timeout timer is unref'd, so a hung predicate on an otherwise + // idle event loop could let the process exit before the fallback lands + // (and node:test on Node 22 tears the loop down early, cancelling the test). + // A ref'd timer keeps the loop alive until the fallback resolves. + return new Promise((resolve) => { + let settled = false + let timer: ReturnType | undefined + const finish = (value: boolean): void => { + if (settled) { + return + } + settled = true + if (timer !== undefined) { + clearTimeout(timer) + } + signal?.removeEventListener('abort', onAbort) + resolve(value) + } + function onAbort(): void { + finish(fallback()) + } + if (hasTimeout) { + timer = setTimeout(() => finish(fallback()), timeoutMs) + } + signal?.addEventListener('abort', onAbort, { once: true }) + decided.then(finish, () => finish(fallback())) + }) } const isAbortError = (err: unknown): boolean => @@ -418,11 +496,24 @@ const runAgentLoopInner = async ( break } - if (!shouldCallReplanner(result)) { + // The trigger is host-configurable (replanAfter); a predicate is bounded + // by the same watchdog/abort as an LLM call and falls back to the + // 'failure' rule on error, so it can never stall the run. + const wantReplanner = await replanTriggered(ctx.config.replanAfter, result, { + signal, + timeoutMs: ctx.config.llmTimeoutMs ?? 0, + onError: (err) => + proxiedCtx.emit({ + type: 'log', + level: 'warn', + message: `[runner] replanAfter predicate threw - using the failure rule: ${(err as Error).message}`, + }), + }) + if (!wantReplanner) { proxiedCtx.emit({ type: 'replan.decision', mode: 'continue', - reason: 'step succeeded cleanly, skipping LLM replanner', + reason: 'replan trigger not met, skipping LLM replanner', cause: 'clean-step', }) stepIndex++ diff --git a/src/types.ts b/src/types.ts index 74c72ee..a3e47dc 100644 --- a/src/types.ts +++ b/src/types.ts @@ -111,6 +111,19 @@ export interface IAgentConfig { maxStepsPerTask: number // Cap on the number of "revise" decisions the replanner can make per run. maxRevisions?: number + // What makes the LLM replanner run after a step (it never runs after the + // last planned step): + // 'failure' (default) - the step was blocked, or a tool call failed and + // stayed failed (a later successful call to the same tool within the + // step counts as self-corrected); + // 'always' - after every step, e.g. when the host surfaces problems to + // the replanner through systemPrompt/domain context (costs one extra + // LLM call per step); + // predicate - decides per step result; may be async and close over host + // state. A predicate that throws, rejects, or outlives llmTimeoutMs / + // the run signal falls back to the 'failure' rule, so a buggy or hung + // predicate can never stall the run. + replanAfter?: ReplanTrigger // Soft cap on cumulative tokens; checked between steps and triggers an // early jump to synthesis when crossed. maxTotalTokens?: number @@ -197,6 +210,12 @@ export interface IStepResult { blocked: boolean } +// When the LLM replanner is consulted after a step; see IAgentConfig.replanAfter. +export type ReplanTrigger = + | 'failure' + | 'always' + | ((result: IStepResult) => boolean | Promise) + export type ReplanCause = 'last-step' | 'clean-step' | 'llm-decision' type AgentEventBody = diff --git a/tests/runner.test.ts b/tests/runner.test.ts index f4c3f13..ace7e47 100644 --- a/tests/runner.test.ts +++ b/tests/runner.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict' import test from 'node:test' -import { resolveResume, shouldCallReplanner } from '../src/runner.ts' +import { replanTriggered, resolveResume, shouldCallReplanner } from '../src/runner.ts' import type { IAgentInternalContext } from '../src/internal.ts' import type { IAgentConfig, IPersistence, IRunSnapshot, IStepResult } from '../src/types.ts' @@ -44,6 +44,49 @@ test('shouldCallReplanner ignores summary prose entirely (no language bias)', () ) }) +test('shouldCallReplanner treats a later successful retry of the same tool as resolved', () => { + const fail = { name: 't', input: {}, output: 'err', ok: false } + const okSame = { name: 't', input: {}, output: 'data', ok: true } + const okOther = { name: 'other', input: {}, output: 'data', ok: true } + // Self-corrected within the executor's multi-step loop - no replan. + assert.equal(shouldCallReplanner(result({ toolCalls: [fail, okSame] })), false) + // A success of a DIFFERENT tool does not resolve the failure. + assert.equal(shouldCallReplanner(result({ toolCalls: [fail, okOther] })), true) + // The failure came after the success - still failed at step end. + assert.equal(shouldCallReplanner(result({ toolCalls: [okSame, fail] })), true) +}) + +test("replanTriggered resolves 'failure' | 'always' | predicate", async () => { + const clean = result({ toolCalls: [{ name: 't', input: {}, output: 1, ok: true }] }) + const failed = result({ toolCalls: [{ name: 't', input: {}, output: 'e', ok: false }] }) + assert.equal(await replanTriggered(undefined, clean), false) + assert.equal(await replanTriggered('failure', failed), true) + assert.equal(await replanTriggered('always', clean), true) + // The predicate wins in both directions. + assert.equal(await replanTriggered(async () => true, clean), true) + assert.equal(await replanTriggered(() => false, failed), false) +}) + +test('replanTriggered: a throwing predicate falls back to the failure rule', async () => { + const failed = result({ toolCalls: [{ name: 't', input: {}, output: 'e', ok: false }] }) + const clean = result({}) + const errors: unknown[] = [] + const boom = () => { + throw new Error('boom') + } + assert.equal(await replanTriggered(boom, failed, { onError: (e) => errors.push(e) }), true) + assert.equal(await replanTriggered(boom, clean), false) + assert.equal(errors.length, 1) + assert.match((errors[0] as Error).message, /boom/) +}) + +test('replanTriggered: a hung predicate is bounded by the watchdog', async () => { + const failed = result({ toolCalls: [{ name: 't', input: {}, output: 'e', ok: false }] }) + const never = () => new Promise(() => {}) + // Falls back to the failure rule when the predicate outlives timeoutMs. + assert.equal(await replanTriggered(never, failed, { timeoutMs: 20 }), true) +}) + const baseConfig = (overrides: Partial = {}): IAgentConfig => ({ clientName: 't', providerType: 'openai',