diff --git a/.changeset/log-viewer-perf-colors.md b/.changeset/log-viewer-perf-colors.md new file mode 100644 index 00000000..ad3b35e0 --- /dev/null +++ b/.changeset/log-viewer-perf-colors.md @@ -0,0 +1,14 @@ +--- +"@perstack/tui-components": patch +"@perstack/log": patch +"@perstack/filesystem-storage": patch +--- + +fix: optimize log viewer TUI performance and improve run status colors + +- Remove continueToNextStep and resolveToolResults from tree event loading (~400MB → ~15MB) +- Parallelize run event loading with Promise.all +- Add header-based checkpoint filtering for run-specific queries (25x faster) +- Use getFirstEvent for job query extraction instead of loading all events +- Fix double statSync in checkpoint sort +- Normalize run status colors: primary for completed/suspending, red for errors, muted for incomplete diff --git a/packages/filesystem/src/checkpoint.ts b/packages/filesystem/src/checkpoint.ts index f4aca4ff..24336161 100644 --- a/packages/filesystem/src/checkpoint.ts +++ b/packages/filesystem/src/checkpoint.ts @@ -1,4 +1,12 @@ -import { existsSync, readdirSync, readFileSync, statSync } from "node:fs" +import { + closeSync, + existsSync, + openSync, + readdirSync, + readFileSync, + readSync, + statSync, +} from "node:fs" import { mkdir, readFile, writeFile } from "node:fs/promises" import path from "node:path" import { type Checkpoint, checkpointSchema } from "@perstack/core" @@ -37,8 +45,19 @@ export function getCheckpointsByJobId(jobId: string): Checkpoint[] { return [] } const files = readdirSync(checkpointDir).filter((file) => file.endsWith(".json")) - const checkpoints: Checkpoint[] = [] + // Read mtime once during the initial scan to avoid double statSync calls during sort + const fileEntries: { file: string; mtime: number }[] = [] for (const file of files) { + try { + const mtime = statSync(path.resolve(checkpointDir, file)).mtimeMs + fileEntries.push({ file, mtime }) + } catch { + fileEntries.push({ file, mtime: 0 }) + } + } + fileEntries.sort((a, b) => a.mtime - b.mtime) + const checkpoints: Checkpoint[] = [] + for (const { file } of fileEntries) { try { const content = readFileSync(path.resolve(checkpointDir, file), "utf-8") checkpoints.push(checkpointSchema.parse(JSON.parse(content))) @@ -46,13 +65,46 @@ export function getCheckpointsByJobId(jobId: string): Checkpoint[] { // Ignore invalid checkpoints } } - return checkpoints.sort((a, b) => { + return checkpoints +} + +export function getCheckpointsByRunId(jobId: string, runId: string): Checkpoint[] { + const checkpointDir = getCheckpointDir(jobId) + if (!existsSync(checkpointDir)) { + return [] + } + const files = readdirSync(checkpointDir).filter((file) => file.endsWith(".json")) + // Pre-filter by reading only the first 256 bytes to check runId before full parse + const runIdPattern = `"runId":"${runId}"` + const matchingFiles: { file: string; mtime: number }[] = [] + const headerBuf = Buffer.alloc(256) + for (const file of files) { + const filePath = path.resolve(checkpointDir, file) try { - const aStat = statSync(path.resolve(checkpointDir, `${a.id}.json`)) - const bStat = statSync(path.resolve(checkpointDir, `${b.id}.json`)) - return aStat.mtimeMs - bStat.mtimeMs + const fd = openSync(filePath, "r") + try { + const bytesRead = readSync(fd, headerBuf, 0, 256, 0) + const header = headerBuf.toString("utf-8", 0, bytesRead) + if (header.includes(runIdPattern)) { + const mtime = statSync(filePath).mtimeMs + matchingFiles.push({ file, mtime }) + } + } finally { + closeSync(fd) + } } catch { - return 0 + // Ignore inaccessible files } - }) + } + matchingFiles.sort((a, b) => a.mtime - b.mtime) + const checkpoints: Checkpoint[] = [] + for (const { file } of matchingFiles) { + try { + const content = readFileSync(path.resolve(checkpointDir, file), "utf-8") + checkpoints.push(checkpointSchema.parse(JSON.parse(content))) + } catch { + // Ignore invalid checkpoints + } + } + return checkpoints } diff --git a/packages/filesystem/src/event.ts b/packages/filesystem/src/event.ts index 9ffe7dbd..52aab20c 100644 --- a/packages/filesystem/src/event.ts +++ b/packages/filesystem/src/event.ts @@ -61,6 +61,53 @@ export function getEventContents( return events } +/** + * Get a single event by type from a run (returns the first match by timestamp). + * Much faster than loading all events when you only need one. + */ +export function getFirstEvent( + jobId: string, + runId: string, + typeFilter: string, +): RunEvent | undefined { + const runDir = getRunDir(jobId, runId) + if (!existsSync(runDir)) { + return undefined + } + const files = readdirSync(runDir) + .filter((file) => file.startsWith("event-") && file.includes(`-${typeFilter}.json`)) + .sort() + if (files.length === 0) return undefined + try { + const content = readFileSync(path.resolve(runDir, files[0]), "utf-8") + return JSON.parse(content) as RunEvent + } catch { + return undefined + } +} + +/** + * Get event stats from filenames only (no file content reading). + * Returns total event count and max step number. + */ +export function getEventStats( + jobId: string, + runId: string, +): { eventCount: number; maxStep: number } { + const runDir = getRunDir(jobId, runId) + if (!existsSync(runDir)) { + return { eventCount: 0, maxStep: 0 } + } + const eventFiles = readdirSync(runDir).filter((file) => file.startsWith("event-")) + let maxStep = 0 + for (const file of eventFiles) { + const parts = file.split(".")[0].split("-") + const step = Number(parts[2]) + if (step > maxStep) maxStep = step + } + return { eventCount: eventFiles.length, maxStep } +} + export function getRunIdsByJobId(jobId: string): string[] { const runsDir = path.resolve(getJobDir(jobId), "runs") if (!existsSync(runsDir)) { diff --git a/packages/filesystem/src/index.ts b/packages/filesystem/src/index.ts index db69fd96..13bc0e93 100644 --- a/packages/filesystem/src/index.ts +++ b/packages/filesystem/src/index.ts @@ -4,8 +4,15 @@ export { getCheckpointDir, getCheckpointPath, getCheckpointsByJobId, + getCheckpointsByRunId, } from "./checkpoint.js" -export { defaultStoreEvent, getEventContents, getRunIdsByJobId } from "./event.js" +export { + defaultStoreEvent, + getEventContents, + getEventStats, + getFirstEvent, + getRunIdsByJobId, +} from "./event.js" export { createInitialJob, getAllJobs, diff --git a/packages/log/src/data-fetcher.ts b/packages/log/src/data-fetcher.ts index bc1009d5..6139ff78 100644 --- a/packages/log/src/data-fetcher.ts +++ b/packages/log/src/data-fetcher.ts @@ -6,7 +6,10 @@ import { getAllJobs, getAllRuns, getCheckpointsByJobId, + getCheckpointsByRunId, getEventContents, + getEventStats, + getFirstEvent, getRunIdsByJobId, retrieveJob, } from "@perstack/filesystem-storage" @@ -16,16 +19,23 @@ export interface LogDataFetcher { getLatestJob(): Promise getRuns(jobId: string): Promise getCheckpoints(jobId: string): Promise + getCheckpointsForRun(jobId: string, runId: string): Promise getCheckpoint(jobId: string, checkpointId: string): Promise getEvents(jobId: string, runId: string): Promise getAllEventsForJob(jobId: string): Promise getTreeEventsForJob(jobId: string): Promise + getFirstEventForRun( + jobId: string, + runId: string, + typeFilter: string, + ): Promise } export interface StorageAdapter { getAllJobs(): Promise retrieveJob(jobId: string): Promise getCheckpointsByJobId(jobId: string): Promise + getCheckpointsByRunId(jobId: string, runId: string): Promise retrieveCheckpoint(jobId: string, checkpointId: string): Promise getEventContents( jobId: string, @@ -33,6 +43,8 @@ export interface StorageAdapter { maxStep?: number, typeFilter?: Set, ): Promise + getFirstEvent(jobId: string, runId: string, typeFilter: string): Promise + getEventStats(jobId: string, runId: string): { eventCount: number; maxStep: number } getAllRuns(): Promise getRunIdsByJobId(jobId: string): string[] getJobIds(): string[] @@ -105,6 +117,10 @@ export function createLogDataFetcher(storage: StorageAdapter): LogDataFetcher { return storage.getCheckpointsByJobId(jobId) }, + async getCheckpointsForRun(jobId: string, runId: string): Promise { + return storage.getCheckpointsByRunId(jobId, runId) + }, + async getCheckpoint(jobId: string, checkpointId: string): Promise { return storage.retrieveCheckpoint(jobId, checkpointId) }, @@ -115,11 +131,10 @@ export function createLogDataFetcher(storage: StorageAdapter): LogDataFetcher { async getAllEventsForJob(jobId: string): Promise { const runs = await this.getRuns(jobId) - const allEvents: RunEvent[] = [] - for (const run of runs) { - const events = await storage.getEventContents(jobId, run.runId) - allEvents.push(...events) - } + const results = await Promise.all( + runs.map((run) => storage.getEventContents(jobId, run.runId)), + ) + const allEvents = results.flat() return allEvents.sort((a, b) => a.timestamp - b.timestamp) }, @@ -132,18 +147,23 @@ export function createLogDataFetcher(storage: StorageAdapter): LogDataFetcher { "completeRun", "stopRunByError", "retry", - "continueToNextStep", - "resolveToolResults", ]) // Use getRunIdsByJobId to discover ALL runs (including those without run-setting.json) const runIds = storage.getRunIdsByJobId(jobId) - const allEvents: RunEvent[] = [] - for (const runId of runIds) { - const events = await storage.getEventContents(jobId, runId, undefined, treeEventTypes) - allEvents.push(...events) - } + const results = await Promise.all( + runIds.map((runId) => storage.getEventContents(jobId, runId, undefined, treeEventTypes)), + ) + const allEvents = results.flat() return allEvents.sort((a, b) => a.timestamp - b.timestamp) }, + + async getFirstEventForRun( + jobId: string, + runId: string, + typeFilter: string, + ): Promise { + return storage.getFirstEvent(jobId, runId, typeFilter) + }, } } @@ -165,10 +185,13 @@ export function createStorageAdapter(basePath: string): StorageAdapter { getAllJobs: async () => getAllJobs(), retrieveJob: async (jobId) => retrieveJob(jobId), getCheckpointsByJobId: async (jobId) => getCheckpointsByJobId(jobId), + getCheckpointsByRunId: async (jobId, runId) => getCheckpointsByRunId(jobId, runId), retrieveCheckpoint: async (jobId, checkpointId) => defaultRetrieveCheckpoint(jobId, checkpointId), getEventContents: async (jobId, runId, maxStep, typeFilter) => getEventContents(jobId, runId, maxStep, typeFilter), + getFirstEvent: async (jobId, runId, typeFilter) => getFirstEvent(jobId, runId, typeFilter), + getEventStats: (jobId, runId) => getEventStats(jobId, runId), getAllRuns: async () => getAllRuns(), getRunIdsByJobId: (jobId) => getRunIdsByJobId(jobId), getJobIds: () => { diff --git a/packages/tui-components/src/log-viewer/app.tsx b/packages/tui-components/src/log-viewer/app.tsx index fa6a28ca..e9f2d83b 100644 --- a/packages/tui-components/src/log-viewer/app.tsx +++ b/packages/tui-components/src/log-viewer/app.tsx @@ -32,8 +32,7 @@ async function extractJobQuery(fetcher: LogDataFetcher, job: Job): Promise e.type === "startRun") + const startRunEvent = await fetcher.getFirstEventForRun(job.id, firstRun.runId, "startRun") if (!startRunEvent) return undefined return extractQueryFromStartRun(startRunEvent) } catch { @@ -80,8 +79,7 @@ export const LogViewerApp = ({ fetcher, initialJobId, initialRunId }: LogViewerA setLoading(false) return } - const checkpoints = await fetcher.getCheckpoints(initialJobId) - const runCheckpoints = checkpoints.filter((cp: Checkpoint) => cp.runId === initialRunId) + const runCheckpoints = await fetcher.getCheckpointsForRun(initialJobId, initialRunId) const run: RunInfo = { jobId: initialJobId, runId: initialRunId } setScreen({ type: "checkpointList", job, run, checkpoints: runCheckpoints }) setLoading(false) @@ -162,8 +160,7 @@ export const LogViewerApp = ({ fetcher, initialJobId, initialRunId }: LogViewerA async (job: Job, run: RunInfo) => { setLoading(true) try { - const checkpoints = await fetcher.getCheckpoints(job.id) - const runCheckpoints = checkpoints.filter((cp: Checkpoint) => cp.runId === run.runId) + const runCheckpoints = await fetcher.getCheckpointsForRun(job.id, run.runId) setScreen({ type: "checkpointList", job, run, checkpoints: runCheckpoints }) } catch (err) { setError(err instanceof Error ? err.message : String(err)) diff --git a/packages/tui-components/src/log-viewer/components/log-info-content.tsx b/packages/tui-components/src/log-viewer/components/log-info-content.tsx index eeab8b6a..725831b1 100644 --- a/packages/tui-components/src/log-viewer/components/log-info-content.tsx +++ b/packages/tui-components/src/log-viewer/components/log-info-content.tsx @@ -34,9 +34,10 @@ function getUsageIcon(ratio: number): string { } function statusColor(status: string): string { - if (status === "completed") return colors.success - if (status === "running" || status === "proceeding" || status === "init") return colors.accent - return colors.destructive + if (status === "completed") return colors.primary + if (status === "error") return colors.destructive + if (status === "suspending") return colors.primary + return colors.muted } // --- Job --- diff --git a/packages/tui-components/src/log-viewer/screens/run-list.tsx b/packages/tui-components/src/log-viewer/screens/run-list.tsx index 852a9d8a..91b83c40 100644 --- a/packages/tui-components/src/log-viewer/screens/run-list.tsx +++ b/packages/tui-components/src/log-viewer/screens/run-list.tsx @@ -52,11 +52,11 @@ function statusIcon(status: string): string { function statusColor(status: string): string | undefined { switch (status) { case "completed": - return colors.success + return colors.primary case "error": return colors.destructive - case "running": - return colors.accent + case "suspending": + return colors.primary default: return colors.muted }