Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion apps/cli/src/commands/results/serve.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1002,8 +1002,10 @@ function handleConfig(
{ agentvDir, searchDir }: DataContext,
options?: { readOnly?: boolean; projectDashboard?: boolean; currentProjectId?: string },
) {
const config = loadStudioConfig(agentvDir);
return c.json({
...loadStudioConfig(agentvDir),
threshold: config.threshold,
app_name: config.appName,
read_only: options?.readOnly === true,
project_name: path.basename(searchDir),
project_dashboard: options?.projectDashboard === true,
Expand Down
76 changes: 50 additions & 26 deletions apps/cli/src/commands/results/studio-config.ts
Original file line number Diff line number Diff line change
@@ -1,37 +1,39 @@
/**
* Dashboard configuration loader.
*
* Reads dashboard-specific settings from the `dashboard:` section of
* `.agentv/config.yaml`, falling back to the legacy `studio:` section for
* compatibility. Preserves all other fields (required_version,
* eval_patterns, execution, etc.) when saving.
*
* Location: `.agentv/config.yaml`
* Reads dashboard-specific settings from the `dashboard:` section of config.yaml.
* Project-local `.agentv/config.yaml` takes precedence over the global
* `${AGENTV_HOME:-~/.agentv}/config.yaml`, with legacy `studio:` and root-level
* threshold keys still accepted for compatibility. Saving writes only to the
* project-local file and preserves unrelated fields.
*
* config.yaml format:
* required_version: ">=4.2.0"
* dashboard:
* app_name: agentv # displayed in the Dashboard shell
* threshold: 0.8 # score >= this value is considered "pass"
*
* Backward compat: reads `studio.threshold`, `studio.pass_threshold`, and
* root-level `pass_threshold` as fallbacks. On save, always writes `threshold`
* under `dashboard:`.
*
* If no config.yaml exists, defaults are used.
* If no config.yaml exists in either location, defaults are used.
*/

import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import path from 'node:path';

import { DEFAULT_THRESHOLD, parseYamlValue } from '@agentv/core';
import { DEFAULT_THRESHOLD, getAgentvConfigDir, parseYamlValue } from '@agentv/core';
import { stringify as stringifyYaml } from 'yaml';

export interface StudioConfig {
threshold: number;
appName: string;
}

const DEFAULTS: StudioConfig = {
threshold: DEFAULT_THRESHOLD,
appName: 'agentv',
};

/**
Expand All @@ -43,33 +45,42 @@ const DEFAULTS: StudioConfig = {
* Clamps `threshold` to [0, 1].
*/
export function loadStudioConfig(agentvDir: string): StudioConfig {
const configPath = path.join(agentvDir, 'config.yaml');

if (!existsSync(configPath)) {
return { ...DEFAULTS };
}

const raw = readFileSync(configPath, 'utf-8');
const parsed = parseYamlValue(raw);

if (!parsed || typeof parsed !== 'object') {
return { ...DEFAULTS };
}
const localConfigPath = path.join(agentvDir, 'config.yaml');
const globalConfigPath = path.join(getAgentvConfigDir(), 'config.yaml');
const localConfig = loadParsedConfig(localConfigPath);
const globalConfig =
path.resolve(globalConfigPath) === path.resolve(localConfigPath)
? undefined
: loadParsedConfig(globalConfigPath);

// Prefer dashboard config, then legacy studio config, then root-level pass_threshold.
const config = parsed as Record<string, unknown>;
const threshold = [
readThreshold(config.dashboard),
readThreshold(config.studio),
typeof config.pass_threshold === 'number' ? config.pass_threshold : undefined,
readThreshold(localConfig?.dashboard),
readThreshold(localConfig?.studio),
typeof localConfig?.pass_threshold === 'number' ? localConfig.pass_threshold : undefined,
readThreshold(globalConfig?.dashboard),
readThreshold(globalConfig?.studio),
typeof globalConfig?.pass_threshold === 'number' ? globalConfig.pass_threshold : undefined,
DEFAULTS.threshold,
].find((value) => value !== undefined) as number;

return {
threshold: Math.min(1, Math.max(0, threshold)),
appName:
readAppName(localConfig?.dashboard) ??
readAppName(globalConfig?.dashboard) ??
DEFAULTS.appName,
};
}

function loadParsedConfig(configPath: string): Record<string, unknown> | undefined {
if (!existsSync(configPath)) return undefined;

const raw = readFileSync(configPath, 'utf-8');
const parsed = parseYamlValue(raw);
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return undefined;
return parsed as Record<string, unknown>;
}

function readThreshold(section: unknown): number | undefined {
if (!section || typeof section !== 'object' || Array.isArray(section)) return undefined;
const values = section as Record<string, unknown>;
Expand All @@ -78,6 +89,14 @@ function readThreshold(section: unknown): number | undefined {
return undefined;
}

function readAppName(section: unknown): string | undefined {
if (!section || typeof section !== 'object' || Array.isArray(section)) return undefined;
const value = (section as Record<string, unknown>).app_name;
if (typeof value !== 'string') return undefined;
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : undefined;
}

/**
* Save dashboard config to `config.yaml` in the given `.agentv/` directory.
* Merges into the existing file, preserving all non-dashboard fields
Expand Down Expand Up @@ -123,7 +142,12 @@ export function saveStudioConfig(agentvDir: string, config: StudioConfig): void
const { studio: _legacyStudio, ...withoutStudio } = existing;
existing = {
...withoutStudio,
dashboard: { ...studioRest, ...dashboardRest, ...config },
dashboard: {
...studioRest,
...dashboardRest,
threshold: config.threshold,
app_name: config.appName,
},
};

const yamlStr = stringifyYaml(existing);
Expand Down
8 changes: 4 additions & 4 deletions apps/cli/test/commands/results/serve.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ describe('resolveDashboardMode', () => {

const MOCK_STUDIO_HTML = `<!doctype html>
<html lang="en" class="dark">
<head><title>AgentV Dashboard</title></head>
<head><title>agentv</title></head>
<body class="bg-gray-950 text-gray-100"><div id="root"></div></body>
</html>`;

Expand Down Expand Up @@ -262,7 +262,7 @@ describe('serve app', () => {
const res = await app.request('/');
expect(res.status).toBe(200);
const html = await res.text();
expect(html).toContain('AgentV Dashboard');
expect(html).toContain('agentv');
expect(html).toContain('<div id="root">');
});
});
Expand Down Expand Up @@ -455,7 +455,7 @@ describe('serve app', () => {
const res = await app.request('/');
expect(res.status).toBe(200);
const html = await res.text();
expect(html).toContain('AgentV Dashboard');
expect(html).toContain('agentv');
});

it('serves feedback API with empty results', async () => {
Expand Down Expand Up @@ -1069,7 +1069,7 @@ describe('serve app', () => {
const res = await app.request('/runs/some-run');
expect(res.status).toBe(200);
const html = await res.text();
expect(html).toContain('AgentV Dashboard');
expect(html).toContain('agentv');
});

it('returns 404 JSON for unknown API routes', async () => {
Expand Down
73 changes: 68 additions & 5 deletions apps/cli/test/commands/results/studio-config.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';

Expand All @@ -10,18 +10,27 @@ import { loadStudioConfig, saveStudioConfig } from '../../../src/commands/result

describe('loadStudioConfig', () => {
let tempDir: string;
let previousAgentvHome: string | undefined;

beforeEach(() => {
tempDir = mkdtempSync(path.join(tmpdir(), 'studio-config-'));
previousAgentvHome = process.env.AGENTV_HOME;
process.env.AGENTV_HOME = path.join(tempDir, 'home');
});

afterEach(() => {
if (previousAgentvHome === undefined) {
process.env.AGENTV_HOME = undefined;
} else {
process.env.AGENTV_HOME = previousAgentvHome;
}
rmSync(tempDir, { recursive: true, force: true });
});

it('returns defaults when no config.yaml exists', () => {
const config = loadStudioConfig(tempDir);
expect(config.threshold).toBe(DEFAULT_THRESHOLD);
expect(config.appName).toBe('agentv');
});

it.each([
Expand Down Expand Up @@ -53,6 +62,48 @@ describe('loadStudioConfig', () => {
expect(config.threshold).toBe(0.9);
});

it('reads dashboard.app_name for white labelling', () => {
writeFileSync(path.join(tempDir, 'config.yaml'), 'dashboard:\n app_name: ai evals\n');
expect(loadStudioConfig(tempDir).appName).toBe('ai evals');
});

it('ignores blank dashboard.app_name', () => {
writeFileSync(path.join(tempDir, 'config.yaml'), 'dashboard:\n app_name: " "\n');
expect(loadStudioConfig(tempDir).appName).toBe('agentv');
});

it('falls back to global config.yaml for dashboard settings', () => {
const homeDir = process.env.AGENTV_HOME;
if (!homeDir) throw new Error('AGENTV_HOME test setup failed');
mkdirSync(homeDir, { recursive: true });
writeFileSync(
path.join(homeDir, 'config.yaml'),
'dashboard:\n app_name: ai evals\n threshold: 0.6\n',
);

const config = loadStudioConfig(tempDir);
expect(config.appName).toBe('ai evals');
expect(config.threshold).toBe(0.6);
});

it('prefers local config.yaml over global config.yaml', () => {
const homeDir = process.env.AGENTV_HOME;
if (!homeDir) throw new Error('AGENTV_HOME test setup failed');
mkdirSync(homeDir, { recursive: true });
writeFileSync(
path.join(homeDir, 'config.yaml'),
'dashboard:\n app_name: ai evals\n threshold: 0.6\n',
);
writeFileSync(
path.join(tempDir, 'config.yaml'),
'dashboard:\n app_name: local evals\n threshold: 0.9\n',
);

const config = loadStudioConfig(tempDir);
expect(config.appName).toBe('local evals');
expect(config.threshold).toBe(0.9);
});

it('falls back to legacy studio section when dashboard has no threshold', () => {
writeFileSync(
path.join(tempDir, 'config.yaml'),
Expand Down Expand Up @@ -94,12 +145,20 @@ describe('loadStudioConfig', () => {

describe('saveStudioConfig', () => {
let tempDir: string;
let previousAgentvHome: string | undefined;

beforeEach(() => {
tempDir = mkdtempSync(path.join(tmpdir(), 'studio-config-'));
previousAgentvHome = process.env.AGENTV_HOME;
process.env.AGENTV_HOME = path.join(tempDir, 'home');
});

afterEach(() => {
if (previousAgentvHome === undefined) {
process.env.AGENTV_HOME = undefined;
} else {
process.env.AGENTV_HOME = previousAgentvHome;
}
rmSync(tempDir, { recursive: true, force: true });
});

Expand All @@ -108,21 +167,23 @@ describe('saveStudioConfig', () => {
path.join(tempDir, 'config.yaml'),
'required_version: ">=4.2.0"\neval_patterns:\n - "**/*.eval.yaml"\n',
);
saveStudioConfig(tempDir, { threshold: 0.9 });
saveStudioConfig(tempDir, { threshold: 0.9, appName: 'ai evals' });

const raw = readFileSync(path.join(tempDir, 'config.yaml'), 'utf-8');
const parsed = parseYaml(raw) as Record<string, unknown>;
expect(parsed.required_version).toBe('>=4.2.0');
expect(parsed.eval_patterns).toEqual(['**/*.eval.yaml']);
expect((parsed.dashboard as Record<string, unknown>).threshold).toBe(0.9);
expect((parsed.dashboard as Record<string, unknown>).app_name).toBe('ai evals');
expect((parsed.dashboard as Record<string, unknown>).appName).toBeUndefined();
});

it('writes canonical dashboard.threshold and removes legacy threshold fields on save', () => {
writeFileSync(
path.join(tempDir, 'config.yaml'),
'required_version: ">=4.2.0"\npass_threshold: 0.8\ndashboard:\n pass_threshold: 0.6\nstudio:\n theme: dark\n pass_threshold: 0.5\n',
);
saveStudioConfig(tempDir, { threshold: 0.7 });
saveStudioConfig(tempDir, { threshold: 0.7, appName: 'agentv' });

const raw = readFileSync(path.join(tempDir, 'config.yaml'), 'utf-8');
const parsed = parseYaml(raw) as Record<string, unknown>;
Expand All @@ -133,10 +194,12 @@ describe('saveStudioConfig', () => {
expect(dashboard.theme).toBe('dark');
expect(dashboard.pass_threshold).toBeUndefined();
expect(dashboard.threshold).toBe(0.7);
expect(dashboard.app_name).toBe('agentv');
expect(dashboard.appName).toBeUndefined();
});

it('creates config.yaml when it does not exist', () => {
saveStudioConfig(tempDir, { threshold: 0.6 });
saveStudioConfig(tempDir, { threshold: 0.6, appName: 'agentv' });

const raw = readFileSync(path.join(tempDir, 'config.yaml'), 'utf-8');
const parsed = parseYaml(raw) as Record<string, unknown>;
Expand All @@ -145,7 +208,7 @@ describe('saveStudioConfig', () => {

it('creates directory if it does not exist', () => {
const nestedDir = path.join(tempDir, 'nested', '.agentv');
saveStudioConfig(nestedDir, { threshold: 0.5 });
saveStudioConfig(nestedDir, { threshold: 0.5, appName: 'agentv' });

const raw = readFileSync(path.join(nestedDir, 'config.yaml'), 'utf-8');
const parsed = parseYaml(raw) as Record<string, unknown>;
Expand Down
2 changes: 1 addition & 1 deletion apps/dashboard/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>AgentV Dashboard</title>
<title>agentv</title>
</head>
<body class="bg-gray-950 text-gray-100">
<div id="root"></div>
Expand Down
13 changes: 13 additions & 0 deletions apps/dashboard/src/components/BrandName.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import { DEFAULT_APP_NAME } from '~/lib/api';

export function BrandName({ appName }: { appName: string }) {
if (appName !== DEFAULT_APP_NAME) {
return <span className="av-brand-name">{appName}</span>;
}

return (
<span className="av-brand-name">
agent<span className="text-cyan-400">v</span>
</span>
);
}
8 changes: 7 additions & 1 deletion apps/dashboard/src/components/Layout.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@

import { Outlet } from '@tanstack/react-router';

import { DEFAULT_APP_NAME, useStudioConfig } from '~/lib/api';
import { SidebarProvider, useSidebarContext } from '~/lib/sidebar-context';

import { BrandName } from './BrandName';
import { Breadcrumbs } from './Breadcrumbs';
import { Sidebar } from './Sidebar';

Expand All @@ -26,6 +28,8 @@ export function Layout() {

function LayoutInner() {
const { toggle } = useSidebarContext();
const { data: config } = useStudioConfig();
const appName = config?.app_name ?? DEFAULT_APP_NAME;

return (
<div className="flex h-screen overflow-hidden">
Expand Down Expand Up @@ -59,7 +63,9 @@ function LayoutInner() {
<line x1="3" y1="18" x2="21" y2="18" />
</svg>
</button>
<span className="text-sm font-semibold text-white">AgentV Dashboard</span>
<span className="text-sm font-semibold text-white">
<BrandName appName={appName} />
</span>
</header>

<Breadcrumbs />
Expand Down
Loading
Loading