diff --git a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx index dfa63c704..f25cf6fa9 100644 --- a/apps/web/src/content/docs/docs/evaluation/running-evals.mdx +++ b/apps/web/src/content/docs/docs/evaluation/running-evals.mdx @@ -390,9 +390,11 @@ agentv eval --strict evals/my-eval.yaml ## Config File Defaults -Set default execution options so you don't have to pass them on every CLI invocation. Both `.agentv/config.yaml` and `agentv.config.ts` are supported. +Set default execution options so you don't have to pass them on every CLI invocation. Project-local `.agentv/config.yaml`, home/global `$AGENTV_HOME/config.yaml` (or `~/.agentv/config.yaml`), and `agentv.config.ts` are supported. -### YAML config (`.agentv/config.yaml`) +Project-local YAML config takes precedence over home/global YAML config. AgentV uses the first config file it finds; it does not merge project and global YAML files. + +### YAML config (`.agentv/config.yaml` or `$AGENTV_HOME/config.yaml`) ```yaml execution: @@ -423,35 +425,70 @@ export default defineConfig({ The `{timestamp}` placeholder is replaced with an ISO-like timestamp (e.g., `2026-03-05T14-30-00-000Z`) at execution time. -**Precedence:** CLI flags > `.agentv/config.yaml` > `agentv.config.ts` > built-in defaults. +**Precedence:** CLI flags > project-local `.agentv/config.yaml` > home/global `$AGENTV_HOME/config.yaml` (or `~/.agentv/config.yaml`) > `agentv.config.ts` > built-in defaults. ## Environment Variables ### AGENTV_HOME -Override the data directory for heavy runtime artifacts — workspaces, workspace pool, subagents, trace state, git cache, and downloaded dependencies. Lightweight config and cache files (`version-check.json`, `last-config.json`, `projects.yaml`) always stay in `~/.agentv` regardless of this setting. +Override AgentV's lightweight home/config directory. This directory stores files such as `config.yaml`, `projects.yaml`, `version-check.json`, `last-config.json`, and managed helper binaries. ```bash # Linux/macOS -export AGENTV_HOME=/data/agentv +export AGENTV_HOME=/config/agentv # Windows (PowerShell) -$env:AGENTV_HOME = "D:\agentv" +$env:AGENTV_HOME = "D:\agentv-config" # Windows (CMD) -set AGENTV_HOME=D:\agentv +set AGENTV_HOME=D:\agentv-config ``` -When set, AgentV logs `Using AGENTV_HOME: ` on startup to confirm the override is active. +When unset, AgentV uses `~/.agentv`. + +### AGENTV_DATA_DIR + +Override the heavy runtime data directory for workspaces, workspace pool, subagents, trace state, git caches, downloaded dependencies, and results repository clones. If `AGENTV_DATA_DIR` is unset, AgentV stores heavy data in `AGENTV_HOME` (or `~/.agentv`) for backward compatibility. + +```bash +# Linux/macOS +export AGENTV_HOME=/config/agentv +export AGENTV_DATA_DIR=/data/agentv + +# Windows (PowerShell) +$env:AGENTV_HOME = "D:\agentv-config" +$env:AGENTV_DATA_DIR = "E:\agentv-data" + +# Windows (CMD) +set AGENTV_HOME=D:\agentv-config +set AGENTV_DATA_DIR=E:\agentv-data +``` :::tip[Windows long paths] -If you use a custom `AGENTV_HOME` on Windows for large monorepo workspaces, enable long path support: +If you use a custom `AGENTV_DATA_DIR` on Windows for large monorepo workspaces, enable long path support: ```powershell git config --system core.longpaths true ``` Or set the registry key: `HKLM\SYSTEM\CurrentControlSet\Control\FileSystem\LongPathsEnabled = 1` ::: +### Docker directories + +Keep the container user's `HOME`, AgentV config home, and AgentV heavy data directory separate so config files and large runtime artifacts can be mounted independently: + +```bash +docker run --rm \ + --user "$(id -u):$(id -g)" \ + -e HOME=/home/agentv \ + -e AGENTV_HOME=/home/agentv/.agentv \ + -e AGENTV_DATA_DIR=/data/agentv \ + -v agentv-home:/home/agentv/.agentv \ + -v agentv-data:/data/agentv \ + -v "$PWD:/workspace" \ + -w /workspace \ + agentv +``` + ## All Options Run `agentv eval --help` for the full list of options including workers, timeouts, output formats, and trace dumping. diff --git a/docker-compose.yml b/docker-compose.yml index c67c70c18..a17033b91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,10 +7,12 @@ services: environment: - PORT=${PORT:-3117} - HOME=/home/agentv - - AGENTV_HOME=/home/agentv/.agentv/data + - AGENTV_HOME=/home/agentv/.agentv + - AGENTV_DATA_DIR=/data/agentv - ANTHROPIC_API_KEY volumes: - ${AGENTV_HOME_DIR:-./.agentv-docker}:/home/agentv/.agentv + - ${AGENTV_DATA_DIR_HOST:-./.agentv-data-docker}:/data/agentv - ${AGENTV_PROJECTS_DIR:-./examples}:/data/projects/agentv-examples - ${AGENTV_RESULTS_DIR:-./results}:/data/results/agentv-evalresults command: diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 462a79e72..efae9de49 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -1,6 +1,7 @@ import { readFile } from 'node:fs/promises'; import path from 'node:path'; +import { getAgentvConfigDir } from '../../paths.js'; import { interpolateEnv } from '../interpolation.js'; import type { EvalTargetRef, @@ -59,8 +60,12 @@ export type AgentVConfig = { }; /** - * Load optional .agentv/config.yaml configuration file. - * Searches from eval file directory up to repo root. + * Load optional AgentV YAML configuration. + * + * Project-local `.agentv/config.yaml` files are searched from the eval file + * directory up to the repo root. If no project-local config is found, AgentV + * falls back to the home/global config at `${AGENTV_HOME:-~/.agentv}/config.yaml`. + * The first valid file wins; there is intentionally no cross-file merge. */ export async function loadConfig( evalFilePath: string, @@ -75,56 +80,61 @@ export async function loadConfig( continue; } - try { - const rawConfig = await readFile(configPath, 'utf8'); - const parsed = interpolateEnv(parseYamlValue(rawConfig), process.env) as unknown; - - if (!isJsonObject(parsed)) { - logWarning(`Invalid .agentv/config.yaml format at ${configPath}`); - continue; - } - - const config = parsed as AgentVConfig; - - const requiredVersion = (parsed as Record).required_version; - if (requiredVersion !== undefined && typeof requiredVersion !== 'string') { - logWarning(`Invalid required_version in ${configPath}, expected string`); - continue; - } - - const evalPatterns = (config as Record).eval_patterns; - if (evalPatterns !== undefined && !Array.isArray(evalPatterns)) { - logWarning(`Invalid eval_patterns in ${configPath}, expected array`); - continue; - } - - if (Array.isArray(evalPatterns) && !evalPatterns.every((p) => typeof p === 'string')) { - logWarning(`Invalid eval_patterns in ${configPath}, all entries must be strings`); - continue; - } - - const executionDefaults = parseExecutionDefaults( - (parsed as Record).execution, - configPath, - ); - const results = parseResultsConfig((parsed as Record).results, configPath); - const hooks = parseHooksConfig((parsed as Record).hooks, configPath); - - return { - required_version: requiredVersion as string | undefined, - eval_patterns: evalPatterns as readonly string[] | undefined, - execution: executionDefaults, - results, - ...(hooks && { hooks }), - }; - } catch (error) { - logWarning( - `Could not read .agentv/config.yaml at ${configPath}: ${(error as Error).message}`, - ); - } + const config = await readConfigFile(configPath); + if (config) return config; } - return null; + const globalConfigPath = path.join(getAgentvConfigDir(), 'config.yaml'); + return (await fileExists(globalConfigPath)) ? readConfigFile(globalConfigPath) : null; +} + +async function readConfigFile(configPath: string): Promise { + try { + const rawConfig = await readFile(configPath, 'utf8'); + const parsed = interpolateEnv(parseYamlValue(rawConfig), process.env) as unknown; + + if (!isJsonObject(parsed)) { + logWarning(`Invalid config.yaml format at ${configPath}`); + return null; + } + + const config = parsed as AgentVConfig; + + const requiredVersion = (parsed as Record).required_version; + if (requiredVersion !== undefined && typeof requiredVersion !== 'string') { + logWarning(`Invalid required_version in ${configPath}, expected string`); + return null; + } + + const evalPatterns = (config as Record).eval_patterns; + if (evalPatterns !== undefined && !Array.isArray(evalPatterns)) { + logWarning(`Invalid eval_patterns in ${configPath}, expected array`); + return null; + } + + if (Array.isArray(evalPatterns) && !evalPatterns.every((p) => typeof p === 'string')) { + logWarning(`Invalid eval_patterns in ${configPath}, all entries must be strings`); + return null; + } + + const executionDefaults = parseExecutionDefaults( + (parsed as Record).execution, + configPath, + ); + const results = parseResultsConfig((parsed as Record).results, configPath); + const hooks = parseHooksConfig((parsed as Record).hooks, configPath); + + return { + required_version: requiredVersion as string | undefined, + eval_patterns: evalPatterns as readonly string[] | undefined, + execution: executionDefaults, + results, + ...(hooks && { hooks }), + }; + } catch (error) { + logWarning(`Could not read config.yaml at ${configPath}: ${(error as Error).message}`); + return null; + } } /** diff --git a/packages/core/src/evaluation/providers/pi-coding-agent.ts b/packages/core/src/evaluation/providers/pi-coding-agent.ts index 8d6300ec6..8a6a4651b 100644 --- a/packages/core/src/evaluation/providers/pi-coding-agent.ts +++ b/packages/core/src/evaluation/providers/pi-coding-agent.ts @@ -17,7 +17,7 @@ import path from 'node:path'; import { createInterface } from 'node:readline'; import { fileURLToPath, pathToFileURL } from 'node:url'; -import { getAgentvHome } from '../../paths.js'; +import { getAgentvDataDir } from '../../paths.js'; import { recordPiLogEntry } from './pi-log-tracker.js'; import { normalizeAzureSdkBaseUrl, @@ -80,7 +80,7 @@ function findAgentvRoot(): string { } function findManagedSdkInstallRoot(): string { - return path.join(getAgentvHome(), 'deps', 'pi-sdk'); + return path.join(getAgentvDataDir(), 'deps', 'pi-sdk'); } function resolveGlobalNpmRoot(): string | undefined { diff --git a/packages/core/src/evaluation/results-repo.ts b/packages/core/src/evaluation/results-repo.ts index deb69aa98..ca1543cfd 100644 --- a/packages/core/src/evaluation/results-repo.ts +++ b/packages/core/src/evaluation/results-repo.ts @@ -13,7 +13,7 @@ import os from 'node:os'; import path from 'node:path'; import { promisify } from 'node:util'; -import { getAgentvHome } from '../paths.js'; +import { getAgentvDataDir } from '../paths.js'; import type { ResultsConfig } from './loaders/config-loader.js'; const execFileAsync = promisify(execFile); @@ -82,7 +82,7 @@ export function normalizeResultsConfig(config: ResultsConfig): Required Promise, +): Promise { + const previous = process.env[name]; + if (value === undefined) { + process.env[name] = undefined; + } else { + process.env[name] = value; + } + + return fn().finally(() => { + if (previous === undefined) { + process.env[name] = undefined; + } else { + process.env[name] = previous; + } + }); +} + +describe('loadConfig', () => { + it('falls back to AGENTV_HOME/config.yaml when no project-local config exists', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-global-config-')); + try { + const projectDir = path.join(tempDir, 'project'); + const evalDir = path.join(projectDir, 'evals'); + const homeDir = path.join(tempDir, 'home'); + mkdirSync(evalDir, { recursive: true }); + mkdirSync(homeDir, { recursive: true }); + writeFileSync( + path.join(homeDir, 'config.yaml'), + 'eval_patterns:\n - "**/*.global.eval.yaml"\nexecution:\n verbose: true\n', + ); + + await withOptionalEnv('AGENTV_HOME', homeDir, async () => { + const config = await loadConfig(path.join(evalDir, 'suite.eval.yaml'), projectDir); + expect(config?.eval_patterns).toEqual(['**/*.global.eval.yaml']); + expect(config?.execution).toEqual({ verbose: true }); + }); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it('prefers project-local .agentv/config.yaml over AGENTV_HOME/config.yaml', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-local-config-')); + try { + const projectDir = path.join(tempDir, 'project'); + const evalDir = path.join(projectDir, 'evals'); + const localConfigDir = path.join(projectDir, '.agentv'); + const homeDir = path.join(tempDir, 'home'); + mkdirSync(evalDir, { recursive: true }); + mkdirSync(localConfigDir, { recursive: true }); + mkdirSync(homeDir, { recursive: true }); + writeFileSync( + path.join(homeDir, 'config.yaml'), + 'eval_patterns:\n - "**/*.global.eval.yaml"\nexecution:\n verbose: true\n', + ); + writeFileSync( + path.join(localConfigDir, 'config.yaml'), + 'eval_patterns:\n - "**/*.local.eval.yaml"\nexecution:\n keep_workspaces: true\n', + ); + + await withOptionalEnv('AGENTV_HOME', homeDir, async () => { + const config = await loadConfig(path.join(evalDir, 'suite.eval.yaml'), projectDir); + expect(config?.eval_patterns).toEqual(['**/*.local.eval.yaml']); + expect(config?.execution).toEqual({ keep_workspaces: true }); + }); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); +}); + describe('extractTrialsConfig', () => { it('returns undefined when no execution block', () => { const suite: JsonObject = { tests: [] }; diff --git a/packages/core/test/paths.test.ts b/packages/core/test/paths.test.ts index e32806c7e..e3067aa68 100644 --- a/packages/core/test/paths.test.ts +++ b/packages/core/test/paths.test.ts @@ -1,88 +1,82 @@ -import { afterEach, beforeEach, describe, expect, it, spyOn } from 'bun:test'; +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; import os from 'node:os'; import path from 'node:path'; import { - _resetLoggedForTesting, getAgentvConfigDir, + getAgentvDataDir, getAgentvHome, getSubagentsRoot, getTraceStateRoot, + getWorkspacePoolRoot, getWorkspacesRoot, } from '../src/paths.js'; +function setOptionalEnv(name: string, value: string | undefined): void { + if (value === undefined) { + process.env[name] = undefined; + } else { + process.env[name] = value; + } +} + describe('paths', () => { - const originalEnv = process.env.AGENTV_HOME; + const originalAgentvHome = process.env.AGENTV_HOME; + const originalAgentvDataDir = process.env.AGENTV_DATA_DIR; beforeEach(() => { - _resetLoggedForTesting(); process.env.AGENTV_HOME = undefined; + process.env.AGENTV_DATA_DIR = undefined; }); afterEach(() => { - if (originalEnv !== undefined) { - process.env.AGENTV_HOME = originalEnv; - } else { - process.env.AGENTV_HOME = undefined; - } + setOptionalEnv('AGENTV_HOME', originalAgentvHome); + setOptionalEnv('AGENTV_DATA_DIR', originalAgentvDataDir); }); it('returns ~/.agentv when AGENTV_HOME is not set', () => { + expect(getAgentvConfigDir()).toBe(path.join(os.homedir(), '.agentv')); expect(getAgentvHome()).toBe(path.join(os.homedir(), '.agentv')); }); it('treats the string "undefined" as unset', () => { process.env.AGENTV_HOME = 'undefined'; - expect(getAgentvHome()).toBe(path.join(os.homedir(), '.agentv')); - }); - - it('returns custom path when AGENTV_HOME is set', () => { - process.env.AGENTV_HOME = '/custom/agentv'; - expect(getAgentvHome()).toBe('/custom/agentv'); - }); - - it('getWorkspacesRoot returns correct subpath', () => { - expect(getWorkspacesRoot()).toBe(path.join(os.homedir(), '.agentv', 'workspaces')); - }); - - it('getSubagentsRoot returns correct subpath', () => { - expect(getSubagentsRoot()).toBe(path.join(os.homedir(), '.agentv', 'subagents')); - }); - - it('getTraceStateRoot returns correct subpath', () => { - expect(getTraceStateRoot()).toBe(path.join(os.homedir(), '.agentv', 'trace-state')); + process.env.AGENTV_DATA_DIR = 'undefined'; + expect(getAgentvConfigDir()).toBe(path.join(os.homedir(), '.agentv')); + expect(getAgentvDataDir()).toBe(path.join(os.homedir(), '.agentv')); }); - it('convenience functions respect AGENTV_HOME', () => { - process.env.AGENTV_HOME = '/custom/home'; - expect(getWorkspacesRoot()).toBe(path.join('/custom/home', 'workspaces')); - expect(getSubagentsRoot()).toBe(path.join('/custom/home', 'subagents')); - expect(getTraceStateRoot()).toBe(path.join('/custom/home', 'trace-state')); + it('uses AGENTV_HOME as the lightweight config/home directory', () => { + process.env.AGENTV_HOME = '/custom/agentv-home'; + expect(getAgentvConfigDir()).toBe('/custom/agentv-home'); + expect(getAgentvHome()).toBe('/custom/agentv-home'); }); - it('logs once when AGENTV_HOME is set', () => { - process.env.AGENTV_HOME = '/custom/agentv'; - const spy = spyOn(console, 'log').mockImplementation(() => {}); - getAgentvHome(); - getAgentvHome(); - expect(spy).toHaveBeenCalledTimes(1); - expect(spy).toHaveBeenCalledWith('Using AGENTV_HOME: /custom/agentv'); - spy.mockRestore(); + it('defaults heavy data to the config/home directory', () => { + process.env.AGENTV_HOME = '/custom/agentv-home'; + expect(getAgentvDataDir()).toBe('/custom/agentv-home'); }); - it('does not log when AGENTV_HOME is not set', () => { - const spy = spyOn(console, 'log').mockImplementation(() => {}); - getAgentvHome(); - expect(spy).not.toHaveBeenCalled(); - spy.mockRestore(); + it('uses AGENTV_DATA_DIR for heavy data when set', () => { + process.env.AGENTV_HOME = '/custom/agentv-home'; + process.env.AGENTV_DATA_DIR = '/custom/agentv-data'; + expect(getAgentvConfigDir()).toBe('/custom/agentv-home'); + expect(getAgentvDataDir()).toBe('/custom/agentv-data'); }); - it('getAgentvConfigDir always returns ~/.agentv regardless of AGENTV_HOME', () => { - process.env.AGENTV_HOME = '/data/agentv'; - expect(getAgentvConfigDir()).toBe(path.join(os.homedir(), '.agentv')); + it('heavy data helpers use AGENTV_DATA_DIR', () => { + process.env.AGENTV_HOME = '/custom/agentv-home'; + process.env.AGENTV_DATA_DIR = '/custom/agentv-data'; + expect(getWorkspacesRoot()).toBe(path.join('/custom/agentv-data', 'workspaces')); + expect(getSubagentsRoot()).toBe(path.join('/custom/agentv-data', 'subagents')); + expect(getTraceStateRoot()).toBe(path.join('/custom/agentv-data', 'trace-state')); + expect(getWorkspacePoolRoot()).toBe(path.join('/custom/agentv-data', 'workspace-pool')); }); - it('getAgentvConfigDir returns ~/.agentv when AGENTV_HOME is not set', () => { - expect(getAgentvConfigDir()).toBe(path.join(os.homedir(), '.agentv')); + it('heavy data helpers default to ~/.agentv subpaths', () => { + expect(getWorkspacesRoot()).toBe(path.join(os.homedir(), '.agentv', 'workspaces')); + expect(getSubagentsRoot()).toBe(path.join(os.homedir(), '.agentv', 'subagents')); + expect(getTraceStateRoot()).toBe(path.join(os.homedir(), '.agentv', 'trace-state')); + expect(getWorkspacePoolRoot()).toBe(path.join(os.homedir(), '.agentv', 'workspace-pool')); }); }); diff --git a/plugins/agentv-claude-trace/lib/state.ts b/plugins/agentv-claude-trace/lib/state.ts index 5eef11d02..3c9b85a84 100644 --- a/plugins/agentv-claude-trace/lib/state.ts +++ b/plugins/agentv-claude-trace/lib/state.ts @@ -5,7 +5,16 @@ import { join } from 'node:path'; // Mirrors getTraceStateRoot() from packages/core/src/paths.ts — inlined to avoid // adding @agentv/core as a dependency for this lightweight plugin. -const STATE_DIR = join(process.env.AGENTV_HOME ?? join(homedir(), '.agentv'), 'trace-state'); +function readEnvPath(name: string): string | undefined { + const value = process.env[name]; + if (!value || value === 'undefined') return undefined; + return value; +} + +const STATE_DIR = join( + readEnvPath('AGENTV_DATA_DIR') ?? readEnvPath('AGENTV_HOME') ?? join(homedir(), '.agentv'), + 'trace-state', +); export interface SessionState { sessionId: string; diff --git a/scripts/setup-dashboard-deployment.sh b/scripts/setup-dashboard-deployment.sh index 5b9cb9582..ed39fee14 100755 --- a/scripts/setup-dashboard-deployment.sh +++ b/scripts/setup-dashboard-deployment.sh @@ -166,12 +166,15 @@ write_project_config write_project_registry export AGENTV_HOME_DIR="$home_dir" +export AGENTV_DATA_DIR_HOST="${AGENTV_DATA_DIR_HOST:-$home_dir/data}" export AGENTV_PROJECTS_DIR="$project_dir" export AGENTV_RESULTS_DIR="$results_dir" export AGENTV_UID="${AGENTV_UID:-$(id -u)}" export AGENTV_GID="${AGENTV_GID:-$(id -g)}" export PORT="$port" +mkdir -p "$AGENTV_DATA_DIR_HOST" + docker compose -f "$repo_root/docker-compose.yml" config >/dev/null if [[ "$start" -eq 1 ]]; then @@ -183,6 +186,7 @@ Deployment files are ready in $deploy_dir. Start later with: AGENTV_HOME_DIR=$home_dir \\ + AGENTV_DATA_DIR_HOST=${AGENTV_DATA_DIR_HOST} \\ AGENTV_PROJECTS_DIR=$project_dir \\ AGENTV_RESULTS_DIR=$results_dir \\ AGENTV_UID=${AGENTV_UID} \\