diff --git a/packages/types/src/__tests__/learning-memory.test.ts b/packages/types/src/__tests__/learning-memory.test.ts new file mode 100644 index 0000000000..6cea0de8ff --- /dev/null +++ b/packages/types/src/__tests__/learning-memory.test.ts @@ -0,0 +1,71 @@ +// npx vitest run packages/types/src/__tests__/learning-memory.test.ts + +import { + DEFAULT_LEARNING_CONFIG, + EMPTY_LEARNING_STATE, + learningConfigSchema, + learningStateSchema, + memoryContextSchema, + type LearningState, + type MemoryContext, +} from "../index.js" + +describe("learning types", () => { + it("exports the default learning config", () => { + expect(DEFAULT_LEARNING_CONFIG).toMatchObject({ + enabled: false, + reviewOnTurnCount: 10, + reviewOnToolIterationCount: 50, + }) + }) + + it("parses the empty learning state", () => { + const result = learningStateSchema.safeParse(EMPTY_LEARNING_STATE) + + expect(result.success).toBe(true) + expect(result.data).toEqual(EMPTY_LEARNING_STATE) + }) + + it("applies learning config defaults", () => { + const result = learningConfigSchema.parse({}) + + expect(result).toEqual(DEFAULT_LEARNING_CONFIG) + }) + + it("preserves TypeScript inference for learning state", () => { + const state: LearningState = EMPTY_LEARNING_STATE + + expect(state.version).toBe(1) + }) +}) + +describe("memory types", () => { + it("parses a valid memory context", () => { + const input: MemoryContext = { + entries: [], + revision: 0, + generatedAt: Date.now(), + } + + const result = memoryContextSchema.safeParse(input) + + expect(result.success).toBe(true) + expect(result.data).toEqual(input) + }) + + it("rejects more than ten memory entries", () => { + const result = memoryContextSchema.safeParse({ + entries: Array.from({ length: 11 }, (_, index) => ({ + id: `entry-${index}`, + content: "memory", + source: "learning", + createdAt: index, + updatedAt: index, + })), + revision: 0, + generatedAt: Date.now(), + }) + + expect(result.success).toBe(false) + }) +}) diff --git a/packages/types/src/experiment.ts b/packages/types/src/experiment.ts index d7eb0b03d6..fc8cba2fb8 100644 --- a/packages/types/src/experiment.ts +++ b/packages/types/src/experiment.ts @@ -6,7 +6,14 @@ import type { Keys, Equals, AssertEqual } from "./type-fu.js" * ExperimentId */ -export const experimentIds = ["preventFocusDisruption", "imageGeneration", "runSlashCommand", "customTools"] as const +export const experimentIds = [ + "preventFocusDisruption", + "imageGeneration", + "runSlashCommand", + "customTools", + "selfImproving", + "selfImprovingAutoSkills", +] as const export const experimentIdsSchema = z.enum(experimentIds) @@ -21,6 +28,8 @@ export const experimentsSchema = z.object({ imageGeneration: z.boolean().optional(), runSlashCommand: z.boolean().optional(), customTools: z.boolean().optional(), + selfImproving: z.boolean().optional(), + selfImprovingAutoSkills: z.boolean().optional(), }) export type Experiments = z.infer diff --git a/packages/types/src/global-settings.ts b/packages/types/src/global-settings.ts index 3a5a99eb98..650c7c1ae8 100644 --- a/packages/types/src/global-settings.ts +++ b/packages/types/src/global-settings.ts @@ -78,6 +78,10 @@ export const DEFAULT_CHECKPOINT_TIMEOUT_SECONDS = 15 * GlobalSettings */ +export const selfImprovingScopeSchema = z.enum(["workspace", "global"]) + +export type SelfImprovingScope = z.infer + export const globalSettingsSchema = z.object({ currentApiConfigName: z.string().optional(), listApiConfigMeta: z.array(providerSettingsEntrySchema).optional(), @@ -92,6 +96,10 @@ export const globalSettingsSchema = z.object({ imageGenerationProvider: z.enum(["openrouter"]).optional(), openRouterImageApiKey: z.string().optional(), openRouterImageGenerationSelectedModel: z.string().optional(), + memoryBackend: z.enum(["builtin", "agentmemory"]).optional(), + agentMemoryUrl: z.string().optional(), + selfImprovingScope: selfImprovingScopeSchema.optional(), + selfImprovingAutoSkillsScope: selfImprovingScopeSchema.optional(), customCondensingPrompt: z.string().optional(), diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 4f03a7c879..401076b918 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -14,8 +14,11 @@ export * from "./global-settings.js" export * from "./history.js" export * from "./image-generation.js" export * from "./ipc.js" +export * from "./learning.js" +export * from "./marketplace.js" export * from "./mcp.js" export * from "./message.js" +export * from "./memory.js" export * from "./mode.js" export * from "./model.js" export * from "./provider-settings.js" diff --git a/packages/types/src/learning.ts b/packages/types/src/learning.ts new file mode 100644 index 0000000000..16016834e6 --- /dev/null +++ b/packages/types/src/learning.ts @@ -0,0 +1,200 @@ +import { z } from "zod" + +/** + * FeedbackSignal - types of learning observations + */ +export const feedbackSignalSchema = z.enum([ + "USER_CORRECTION", + "TASK_SUCCESS", + "TASK_FAILURE", + "PATTERN_REPEAT", + "CODE_INDEX_HIT", + "PROMPT_QUALITY", +]) + +export type FeedbackSignal = z.infer + +/** + * LearningConfig - configuration for the learning system + */ +export const learningConfigSchema = z.object({ + enabled: z.boolean().default(false), + reviewOnTurnCount: z.number().int().min(1).default(10), + reviewOnToolIterationCount: z.number().int().min(1).default(50), + maxStoredPatterns: z.number().int().min(1).default(100), + maxStoredEvents: z.number().int().min(1).default(500), + maxPromptPatterns: z.number().int().min(1).default(5), + curatorEnabled: z.boolean().default(true), + curatorIntervalMs: z.number().int().min(60000).default(3600000), + staleAfterDays: z.number().int().min(1).default(14), + archiveAfterDays: z.number().int().min(1).default(60), + codeIndexCorrelationEnabled: z.boolean().default(true), +}) + +export type LearningConfig = z.infer + +export const DEFAULT_LEARNING_CONFIG: LearningConfig = { + enabled: false, + reviewOnTurnCount: 10, + reviewOnToolIterationCount: 50, + maxStoredPatterns: 100, + maxStoredEvents: 500, + maxPromptPatterns: 5, + curatorEnabled: true, + curatorIntervalMs: 3600000, + staleAfterDays: 14, + archiveAfterDays: 60, + codeIndexCorrelationEnabled: true, +} + +/** + * LearningEvent - a single learning observation + */ +export const learningEventSchema = z.object({ + id: z.string(), + signal: feedbackSignalSchema, + timestamp: z.number(), + taskId: z.string().optional(), + workspacePath: z.string().optional(), + mode: z.string().optional(), + context: z.object({ + userTurnCount: z.number().optional(), + toolIterationCount: z.number().optional(), + toolNames: z.array(z.string()).optional(), + promptFingerprint: z.string().optional(), + errorKey: z.string().optional(), + codeIndex: z + .object({ + available: z.boolean(), + hits: z.number(), + topScore: z.number().optional(), + }) + .optional(), + }), + outcome: z.object({ + success: z.boolean().optional(), + corrected: z.boolean().optional(), + summary: z.string().optional(), + confidenceDelta: z.number().optional(), + }), +}) + +export type LearningEvent = z.infer + +/** + * PatternState - lifecycle state for learned patterns + */ +export const patternStateSchema = z.enum(["active", "stale", "archived"]) + +export type PatternState = z.infer + +/** + * PatternType - category of learned pattern + */ +export const patternTypeSchema = z.enum(["prompt", "tool", "error", "skill", "code-index"]) + +export type PatternType = z.infer + +/** + * LearnedPattern - a pattern extracted from learning events + */ +export const learnedPatternSchema = z.object({ + id: z.string(), + patternType: patternTypeSchema, + state: patternStateSchema, + summary: z.string(), + confidenceScore: z.number().min(0).max(1), + frequency: z.number().int().min(0), + successRate: z.number().min(0).max(1), + firstSeenAt: z.number(), + lastSeenAt: z.number(), + lastAppliedAt: z.number().optional(), + sourceSignals: z.array(feedbackSignalSchema), + context: z.object({ + toolNames: z.array(z.string()).optional(), + errorKeys: z.array(z.string()).optional(), + modes: z.array(z.string()).optional(), + workspacePaths: z.array(z.string()).optional(), + }), +}) + +export type LearnedPattern = z.infer + +/** + * ActionType - types of improvement actions + */ +export const actionTypeSchema = z.enum([ + "PROMPT_ENRICHMENT", + "TOOL_PREFERENCE", + "ERROR_AVOIDANCE", + "SKILL_SUGGESTION", + "SKILL_CREATE", + "SKILL_UPDATE", +]) + +export type ActionType = z.infer + +/** + * ImprovementAction - an action to apply based on learned patterns + */ +export const improvementActionSchema = z.object({ + id: z.string(), + actionType: actionTypeSchema, + target: z.enum(["system-prompt", "task-execution", "skills-manager", "review-queue"]), + payload: z.record(z.string(), z.unknown()), + timestamp: z.number(), +}) + +export type ImprovementAction = z.infer + +/** + * LearningTelemetry - telemetry counters for the learning system + */ +export const learningTelemetrySchema = z.object({ + promptEnrichmentUses: z.number().int().default(0), + toolPreferenceUses: z.number().int().default(0), + errorAvoidanceUses: z.number().int().default(0), + skillSuggestionCount: z.number().int().default(0), + lastReviewAt: z.number().optional(), + lastCuratorRunAt: z.number().optional(), +}) + +export type LearningTelemetry = z.infer + +/** + * LearningState - full serializable state of the learning system + */ +export const learningStateSchema = z.object({ + version: z.literal(1), + config: learningConfigSchema, + counters: z.object({ + userTurnsSinceReview: z.number().int().default(0), + toolIterationsSinceReview: z.number().int().default(0), + }), + patterns: z.array(learnedPatternSchema).default([]), + archivedPatterns: z.array(learnedPatternSchema).default([]), + recentEvents: z.array(learningEventSchema).default([]), + pendingActions: z.array(improvementActionSchema).default([]), + telemetry: learningTelemetrySchema, +}) + +export type LearningState = z.infer + +export const EMPTY_LEARNING_STATE: LearningState = { + version: 1, + config: DEFAULT_LEARNING_CONFIG, + counters: { + userTurnsSinceReview: 0, + toolIterationsSinceReview: 0, + }, + patterns: [], + archivedPatterns: [], + recentEvents: [], + pendingActions: [], + telemetry: { + promptEnrichmentUses: 0, + toolPreferenceUses: 0, + errorAvoidanceUses: 0, + skillSuggestionCount: 0, + }, +} diff --git a/packages/types/src/memory.ts b/packages/types/src/memory.ts new file mode 100644 index 0000000000..9cd191b670 --- /dev/null +++ b/packages/types/src/memory.ts @@ -0,0 +1,29 @@ +import { z } from "zod" + +/** + * MemoryEntry - a single durable memory entry for prompt-facing context + * Adapted from Hermes' bounded memory store concept. + */ +export const memoryEntrySchema = z.object({ + id: z.string(), + content: z.string().max(2000), + source: z.enum(["learning", "user", "system", "review"]), + createdAt: z.number(), + updatedAt: z.number(), + relevanceScore: z.number().min(0).max(1).optional(), + tags: z.array(z.string()).optional(), + expiresAt: z.number().optional(), +}) + +export type MemoryEntry = z.infer + +/** + * MemoryContext - bounded set of memory entries for prompt injection + */ +export const memoryContextSchema = z.object({ + entries: z.array(memoryEntrySchema).max(10), + revision: z.number().int().default(0), + generatedAt: z.number(), +}) + +export type MemoryContext = z.infer diff --git a/packages/types/src/vscode-extension-host.ts b/packages/types/src/vscode-extension-host.ts index c09f22aed7..ee5cf7053e 100644 --- a/packages/types/src/vscode-extension-host.ts +++ b/packages/types/src/vscode-extension-host.ts @@ -98,6 +98,7 @@ export interface ExtensionMessage { | "branchWorktreeIncludeResult" | "folderSelected" | "skills" + | "skillsUpdated" | "fileContent" text?: string /** For fileContent: { path, content, error? } */ @@ -290,6 +291,10 @@ export type ExtensionState = Pick< | "includeDiagnosticMessages" | "maxDiagnosticMessages" | "imageGenerationProvider" + | "memoryBackend" + | "agentMemoryUrl" + | "selfImprovingScope" + | "selfImprovingAutoSkillsScope" | "openRouterImageGenerationSelectedModel" | "includeTaskHistoryInEnhance" | "reasoningBlockCollapsed" @@ -360,7 +365,36 @@ export type ExtensionState = Pick< profileThresholds: Record hasOpenedModeSelector: boolean openRouterImageApiKey?: string + memoryBackend?: "builtin" | "agentmemory" + agentMemoryUrl?: string + selfImprovingScope?: "workspace" | "global" + selfImprovingAutoSkillsScope?: "workspace" | "global" messageQueue?: QueuedMessage[] + selfImprovingStatus?: { + enabled: boolean + started: boolean + patternCount: number + eventCount: number + actionCount: number + memoryEntries: number + memoryBackend?: string + skillRecords: number + curatorStatus: { + lastRunAt: number + firstRunDone: boolean + config: { + intervalMs: number + minIdleMs: number + firstRunDeferred: boolean + staleAfterDays: number + archiveAfterDays: number + backupsEnabled: boolean + maxBackups: number + } + } + lastReviewAt?: number + lastCuratorRunAt?: number + } lastShownAnnouncementId?: string apiModelId?: string mcpServers?: McpServer[] diff --git a/src/__tests__/extension.spec.ts b/src/__tests__/extension.spec.ts index e01c739edc..c51c27fb34 100644 --- a/src/__tests__/extension.spec.ts +++ b/src/__tests__/extension.spec.ts @@ -184,6 +184,7 @@ vi.mock("../core/webview/ClineProvider", async () => { postStateToWebview: vi.fn(), postStateToWebviewWithoutClineMessages: vi.fn(), getState: vi.fn().mockResolvedValue({}), + initializeSelfImproving: vi.fn().mockResolvedValue(undefined), initializeCloudProfileSyncWhenReady: vi.fn().mockResolvedValue(undefined), providerSettingsManager: {}, contextProxy: { getGlobalState: vi.fn() }, @@ -261,6 +262,18 @@ describe("extension.ts", () => { expect(dotenvx.config).toHaveBeenCalledTimes(1) }) + test("initializes self-improving through the provider during activation", async () => { + vi.resetModules() + vi.clearAllMocks() + + const { ClineProvider } = await import("../core/webview/ClineProvider") + const { activate } = await import("../extension") + await activate(mockContext) + + const provider = (ClineProvider as any).getVisibleInstance() + expect(provider.initializeSelfImproving).toHaveBeenCalledTimes(1) + }) + describe("cloud auth state handling", () => { beforeEach(() => { vi.resetModules() diff --git a/src/core/config/__tests__/ContextProxy.spec.ts b/src/core/config/__tests__/ContextProxy.spec.ts index 0a24141155..6f07feb901 100644 --- a/src/core/config/__tests__/ContextProxy.spec.ts +++ b/src/core/config/__tests__/ContextProxy.spec.ts @@ -255,6 +255,30 @@ describe("ContextProxy", () => { const storedValue = proxy.getGlobalState("apiModelId") expect(storedValue).toBe("gpt-4") }) + + it("should persist self-improving memory backend settings in global settings", async () => { + const updateGlobalStateSpy = vi.spyOn(proxy, "updateGlobalState") + + await proxy.setValue("memoryBackend", "agentmemory") + await proxy.setValue("agentMemoryUrl", "http://agentmemory.internal:4001") + + expect(updateGlobalStateSpy).toHaveBeenCalledWith("memoryBackend", "agentmemory") + expect(updateGlobalStateSpy).toHaveBeenCalledWith("agentMemoryUrl", "http://agentmemory.internal:4001") + expect(proxy.getGlobalState("memoryBackend")).toBe("agentmemory") + expect(proxy.getGlobalState("agentMemoryUrl")).toBe("http://agentmemory.internal:4001") + }) + + it("should persist self-improving scope settings in global settings", async () => { + const updateGlobalStateSpy = vi.spyOn(proxy, "updateGlobalState") + + await proxy.setValue("selfImprovingScope", "workspace") + await proxy.setValue("selfImprovingAutoSkillsScope", "global") + + expect(updateGlobalStateSpy).toHaveBeenCalledWith("selfImprovingScope", "workspace") + expect(updateGlobalStateSpy).toHaveBeenCalledWith("selfImprovingAutoSkillsScope", "global") + expect(proxy.getGlobalState("selfImprovingScope")).toBe("workspace") + expect(proxy.getGlobalState("selfImprovingAutoSkillsScope")).toBe("global") + }) }) describe("setValues", () => { diff --git a/src/core/prompts/__tests__/system-prompt.spec.ts b/src/core/prompts/__tests__/system-prompt.spec.ts index f555daba06..1b649986ba 100644 --- a/src/core/prompts/__tests__/system-prompt.spec.ts +++ b/src/core/prompts/__tests__/system-prompt.spec.ts @@ -49,6 +49,7 @@ import { ModeConfig } from "@roo-code/types" import { SYSTEM_PROMPT } from "../system" import { McpHub } from "../../../services/mcp/McpHub" +import type { SelfImprovingManager } from "../../../services/self-improving" import { defaultModeSlug, modes, Mode } from "../../../shared/modes" import "../../../utils/path" import { addCustomInstructions } from "../sections/custom-instructions" @@ -272,6 +273,36 @@ describe("SYSTEM_PROMPT", () => { expect(prompt).toMatchFileSnapshot("./__snapshots__/system-prompt/with-undefined-mcp-hub.snap") }) + it("should include learned guidance before rules when available", async () => { + const selfImprovingManager = { + getPromptContextString: () => "\n## Learned Guidance\n- [prompt] Search relevant code before editing\n", + } as unknown as SelfImprovingManager + + const prompt = await SYSTEM_PROMPT( + mockContext, + "/test/path", + false, + undefined, // mcpHub + undefined, // diffStrategy + defaultModeSlug, // mode + undefined, // customModePrompts + undefined, // customModes + undefined, // globalCustomInstructions + experiments, + undefined, // language + undefined, // rooIgnoreInstructions + undefined, // settings + undefined, // todoList + undefined, // modelId + undefined, // skillsManager + selfImprovingManager, + ) + + expect(prompt).toContain("## Learned Guidance") + expect(prompt).toContain("- [prompt] Search relevant code before editing") + expect(prompt.indexOf("## Learned Guidance")).toBeLessThan(prompt.indexOf("====\n\nRULES")) + }) + it("should include vscode language in custom instructions", async () => { // Mock vscode.env.language const vscode = vi.mocked(await import("vscode")) as any diff --git a/src/core/prompts/system.ts b/src/core/prompts/system.ts index 0d6071644a..51ace9a039 100644 --- a/src/core/prompts/system.ts +++ b/src/core/prompts/system.ts @@ -10,6 +10,7 @@ import { isEmpty } from "../../utils/object" import { McpHub } from "../../services/mcp/McpHub" import { CodeIndexManager } from "../../services/code-index/manager" import { SkillsManager } from "../../services/skills/SkillsManager" +import { SelfImprovingManager } from "../../services/self-improving" import type { SystemPromptSettings } from "./types" import { @@ -55,6 +56,7 @@ async function generatePrompt( todoList?: TodoItem[], modelId?: string, skillsManager?: SkillsManager, + selfImprovingManager?: SelfImprovingManager, ): Promise { if (!context) { throw new Error("Extension context is required for generating system prompt") @@ -79,6 +81,9 @@ async function generatePrompt( getSkillsSection(skillsManager, mode as string), ]) + // Inject learned guidance from self-improving system (experiment-gated) + const learningContext = selfImprovingManager?.getPromptContextString() || "" + // Tools catalog is not included in the system prompt. const toolsCatalog = "" @@ -93,7 +98,7 @@ ${getSharedToolUseSection()}${toolsCatalog} ${getCapabilitiesSection(cwd, shouldIncludeMcp ? mcpHub : undefined)} ${modesSection} -${skillsSection ? `\n${skillsSection}` : ""} +${skillsSection ? `\n${skillsSection}` : ""}${learningContext} ${getRulesSection(cwd, settings)} ${getSystemInfoSection(cwd)} @@ -126,6 +131,7 @@ export const SYSTEM_PROMPT = async ( todoList?: TodoItem[], modelId?: string, skillsManager?: SkillsManager, + selfImprovingManager?: SelfImprovingManager, ): Promise => { if (!context) { throw new Error("Extension context is required for generating system prompt") @@ -154,5 +160,6 @@ export const SYSTEM_PROMPT = async ( todoList, modelId, skillsManager, + selfImprovingManager, ) } diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index fcdfd0263d..f8253cc7b4 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -1451,6 +1451,7 @@ export class Task extends EventEmitter implements TaskLike { } const provider = this.providerRef.deref() + const shouldRecordCorrection = !!this.taskAsk if (provider) { if (mode) { @@ -1468,6 +1469,19 @@ export class Task extends EventEmitter implements TaskLike { } } + if (shouldRecordCorrection) { + void provider + .getSelfImprovingManager?.() + ?.recordUserCorrection({ + taskId: this.taskId, + success: false, + corrected: true, + }) + .catch((error: unknown) => { + console.error("[Task#submitUserMessage] Failed to record user correction:", error) + }) + } + this.emit(RooCodeEventName.TaskUserMessage, this.taskId) // Handle the message directly instead of routing through the webview. @@ -3699,6 +3713,7 @@ export class Task extends EventEmitter implements TaskLike { undefined, // todoList this.api.getModel().id, provider.getSkillsManager(), + provider.getSelfImprovingManager(), ) })() } diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index 6a65c858f9..e8e59734fe 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -1485,6 +1485,64 @@ describe("Cline", () => { // Restore console.error consoleErrorSpy.mockRestore() }) + + it("records a user correction when replying to an interactive ask", async () => { + const recordUserCorrection = vi.fn().mockResolvedValue(undefined) + mockProvider.getSelfImprovingManager = vi.fn().mockReturnValue({ recordUserCorrection }) + + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "initial task", + startTask: false, + }) + + const handleResponseSpy = vi.spyOn(task, "handleWebviewAskResponse") + ;(task as any).interactiveAsk = { type: "ask", text: "Need clarification" } + + await task.submitUserMessage("here is the correction") + + expect(recordUserCorrection).toHaveBeenCalledWith({ + taskId: task.taskId, + success: false, + corrected: true, + }) + expect(handleResponseSpy).toHaveBeenCalledWith("messageResponse", "here is the correction", []) + }) + + it("does not wait for correction telemetry before handling the user response", async () => { + let resolveCorrection: (() => void) | undefined + const recordUserCorrection = vi.fn( + () => + new Promise((resolve) => { + resolveCorrection = resolve + }), + ) + mockProvider.getSelfImprovingManager = vi.fn().mockReturnValue({ recordUserCorrection }) + + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "initial task", + startTask: false, + }) + + const handleResponseSpy = vi.spyOn(task, "handleWebviewAskResponse") + ;(task as any).interactiveAsk = { type: "ask", text: "Need clarification" } + + const submitPromise = task.submitUserMessage("here is the correction") + await Promise.resolve() + + expect(recordUserCorrection).toHaveBeenCalledWith({ + taskId: task.taskId, + success: false, + corrected: true, + }) + expect(handleResponseSpy).toHaveBeenCalledWith("messageResponse", "here is the correction", []) + + resolveCorrection?.() + await submitPromise + }) }) }) diff --git a/src/core/tools/CodebaseSearchTool.ts b/src/core/tools/CodebaseSearchTool.ts index f0d906fabd..9191427e6c 100644 --- a/src/core/tools/CodebaseSearchTool.ts +++ b/src/core/tools/CodebaseSearchTool.ts @@ -71,6 +71,11 @@ export class CodebaseSearchTool extends BaseTool<"codebase_search"> { } const searchResults: VectorStoreSearchResult[] = await manager.searchIndex(query, directoryPrefix) + await task.providerRef.deref()?.getSelfImprovingManager?.()?.recordCodeIndexEvent(task.taskId, { + available: true, + hits: searchResults.length, + topScore: searchResults[0]?.score, + }) if (!searchResults || searchResults.length === 0) { pushToolResult(`No relevant code snippets found for the query: "${query}"`) diff --git a/src/core/tools/__tests__/CodebaseSearchTool.spec.ts b/src/core/tools/__tests__/CodebaseSearchTool.spec.ts new file mode 100644 index 0000000000..f6aae41659 --- /dev/null +++ b/src/core/tools/__tests__/CodebaseSearchTool.spec.ts @@ -0,0 +1,68 @@ +import * as vscode from "vscode" + +import { CodebaseSearchTool } from "../CodebaseSearchTool" +import { CodeIndexManager } from "../../../services/code-index/manager" + +vi.mock("vscode", () => ({ + workspace: { + asRelativePath: vi.fn((filePath: string) => filePath.replace("/workspace/", "")), + }, +})) + +vi.mock("../../../services/code-index/manager", () => ({ + CodeIndexManager: { + getInstance: vi.fn(), + }, +})) + +describe("CodebaseSearchTool", () => { + it("records self-improving code index hit details from search results", async () => { + const recordCodeIndexEvent = vi.fn().mockResolvedValue(undefined) + const getSelfImprovingManager = vi.fn().mockReturnValue({ recordCodeIndexEvent }) + const searchIndex = vi.fn().mockResolvedValue([ + { + payload: { + filePath: "/workspace/src/example.ts", + startLine: 10, + endLine: 20, + codeChunk: "const answer = 42", + }, + score: 0.87, + }, + ]) + vi.mocked(CodeIndexManager.getInstance).mockReturnValue({ + isFeatureEnabled: true, + isFeatureConfigured: true, + searchIndex, + } as any) + + const task = { + cwd: "/workspace", + taskId: "task-1", + consecutiveMistakeCount: 0, + providerRef: { + deref: vi.fn().mockReturnValue({ + context: {}, + getSelfImprovingManager, + }), + }, + say: vi.fn().mockResolvedValue(undefined), + } as any + const callbacks = { + askApproval: vi.fn().mockResolvedValue(true), + handleError: vi.fn().mockResolvedValue(undefined), + pushToolResult: vi.fn(), + } + + const tool = new CodebaseSearchTool() + await tool.execute({ query: "find the answer" }, task, callbacks) + + expect(recordCodeIndexEvent).toHaveBeenCalledWith("task-1", { + available: true, + hits: 1, + topScore: 0.87, + }) + expect(callbacks.handleError).not.toHaveBeenCalled() + expect(vscode.workspace.asRelativePath).toHaveBeenCalledWith("/workspace/src/example.ts", false) + }) +}) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 3f5af94cae..0c4979512c 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -75,6 +75,7 @@ import { ShadowCheckpointService } from "../../services/checkpoints/ShadowCheckp import { CodeIndexManager } from "../../services/code-index/manager" import type { IndexProgressUpdate } from "../../services/code-index/interfaces/manager" import { MdmService } from "../../services/mdm/MdmService" +import { SelfImprovingManager } from "../../services/self-improving" import { SkillsManager } from "../../services/skills/SkillsManager" import { fileExistsAtPath } from "../../utils/fs" @@ -163,6 +164,7 @@ export class ClineProvider public readonly latestAnnouncementId = "may-2026-v3.55.0-mimo-handoff-stability" // v3.55.0 Xiaomi MiMo, upstream handoff updates, stability fixes public readonly providerSettingsManager: ProviderSettingsManager public readonly customModesManager: CustomModesManager + public readonly selfImprovingManager: SelfImprovingManager constructor( readonly context: vscode.ExtensionContext, @@ -226,6 +228,32 @@ export class ClineProvider this.log(`Failed to initialize Skills Manager: ${error}`) }) + // Initialize Self-Improving Manager (experiment-gated, zero overhead when disabled) + this.selfImprovingManager = new SelfImprovingManager({ + globalStoragePath: this.contextProxy.globalStorageUri.fsPath, + logger: { + appendLine: (message: string) => this.log(message), + }, + getExperiments: () => this.getGlobalStateSafe("experiments"), + getMemoryBackend: () => this.getGlobalStateSafe("memoryBackend"), + getAgentMemoryUrl: () => this.getGlobalStateSafe("agentMemoryUrl"), + getSelfImprovingScope: () => this.getGlobalStateSafe("selfImprovingScope"), + getAutoSkillsScope: () => this.getGlobalStateSafe("selfImprovingAutoSkillsScope"), + getWorkspacePath: () => this.currentWorkspacePath, + skillsManager: this.skillsManager, + getCodeIndexInfo: () => { + const manager = this.codeIndexManager + if (!manager) { + return { available: false, hits: 0 } + } + + return { + available: true, + hits: 0, // Simplified - actual hit count would come from code index events + } + }, + }) + this.marketplaceManager = new MarketplaceManager(this.context, this.customModesManager) // Forward task events to the provider. @@ -233,10 +261,44 @@ export class ClineProvider this.taskCreationCallback = (instance: Task) => { this.emit(RooCodeEventName.TaskCreated, instance) + const recordTaskCompletionForLearning = (success: boolean) => { + void instance + .getTaskMode() + .catch(() => defaultModeSlug) + .then((mode) => + this.selfImprovingManager.recordTaskCompletion({ + taskId: instance.taskId, + mode, + workspacePath: this.currentWorkspacePath, + success, + ...(success + ? { + toolNames: instance.toolUsage ? Object.keys(instance.toolUsage) : undefined, + } + : {}), + }), + ) + .catch((error) => { + this.log( + `[SelfImproving] recordTaskCompletion error: ${error instanceof Error ? error.message : String(error)}`, + ) + }) + } + // Create named listener functions so we can remove them later. const onTaskStarted = () => this.emit(RooCodeEventName.TaskStarted, instance.taskId) const onTaskCompleted = (taskId: string, tokenUsage: TokenUsage, toolUsage: ToolUsage) => { this.emit(RooCodeEventName.TaskCompleted, taskId, tokenUsage, toolUsage) + + // Feed task completion into self-improving system + recordTaskCompletionForLearning(true) + } + const onTaskUserMessageForLearning = (_taskId: string) => { + this.selfImprovingManager.recordUserTurn().catch((error) => { + this.log( + `[SelfImproving] recordUserTurn error: ${error instanceof Error ? error.message : String(error)}`, + ) + }) } const onTaskAborted = async () => { this.emit(RooCodeEventName.TaskAborted, instance.taskId) @@ -266,6 +328,9 @@ export class ClineProvider }`, ) } + + // Feed task abortion into self-improving system + recordTaskCompletionForLearning(false) } const onTaskFocused = () => this.emit(RooCodeEventName.TaskFocused, instance.taskId) const onTaskUnfocused = () => this.emit(RooCodeEventName.TaskUnfocused, instance.taskId) @@ -283,6 +348,7 @@ export class ClineProvider // Attach the listeners. instance.on(RooCodeEventName.TaskStarted, onTaskStarted) instance.on(RooCodeEventName.TaskCompleted, onTaskCompleted) + instance.on(RooCodeEventName.TaskUserMessage, onTaskUserMessageForLearning) instance.on(RooCodeEventName.TaskAborted, onTaskAborted) instance.on(RooCodeEventName.TaskFocused, onTaskFocused) instance.on(RooCodeEventName.TaskUnfocused, onTaskUnfocused) @@ -300,6 +366,7 @@ export class ClineProvider this.taskEventListeners.set(instance, [ () => instance.off(RooCodeEventName.TaskStarted, onTaskStarted), () => instance.off(RooCodeEventName.TaskCompleted, onTaskCompleted), + () => instance.off(RooCodeEventName.TaskUserMessage, onTaskUserMessageForLearning), () => instance.off(RooCodeEventName.TaskAborted, onTaskAborted), () => instance.off(RooCodeEventName.TaskFocused, onTaskFocused), () => instance.off(RooCodeEventName.TaskUnfocused, onTaskUnfocused), @@ -394,6 +461,15 @@ export class ClineProvider this.log("Cloud profile synchronization is disabled in compatibility mode") } + /** + * Initialize the self-improving manager. + * Called from extension activation after provider construction. + * No-op when experiment is disabled (zero overhead guarantee). + */ + async initializeSelfImproving(): Promise { + await this.selfImprovingManager.initialize() + } + // Adds a new Task instance to clineStack, marking the start of a new task. // The instance is pushed to the top of the stack (LIFO order). // When the task is completed, the top instance is removed, reactivating the @@ -603,6 +679,7 @@ export class ClineProvider this.mcpHub = undefined await this.skillsManager?.dispose() this.skillsManager = undefined + await this.selfImprovingManager.dispose() this.marketplaceManager?.cleanup() this.customModesManager?.dispose() this.taskHistoryStore.dispose() @@ -1864,8 +1941,18 @@ export class ClineProvider await this.postStateToWebview() } + private getGlobalStateSafe(key: K): GlobalState[K] | undefined { + return this.contextProxy.getGlobalState(key) + } + async refreshWorkspace() { + const previousWorkspacePath = this.currentWorkspacePath this.currentWorkspacePath = getWorkspacePath() + + if (previousWorkspacePath !== this.currentWorkspacePath) { + await this.selfImprovingManager.onSettingsChanged(this.getGlobalStateSafe("experiments")) + } + await this.postStateToWebview() } @@ -2094,6 +2181,10 @@ export class ClineProvider imageGenerationProvider, openRouterImageApiKey, openRouterImageGenerationSelectedModel, + memoryBackend, + agentMemoryUrl, + selfImprovingScope, + selfImprovingAutoSkillsScope, lockApiConfigAcrossModes, } = await this.getState() @@ -2264,6 +2355,7 @@ export class ClineProvider followupAutoApproveTimeoutMs: followupAutoApproveTimeoutMs ?? 60000, includeDiagnosticMessages: includeDiagnosticMessages ?? true, maxDiagnosticMessages: maxDiagnosticMessages ?? 50, + selfImprovingStatus: await this.selfImprovingManager.getStatus(), includeTaskHistoryInEnhance: includeTaskHistoryInEnhance ?? true, includeCurrentTime: includeCurrentTime ?? true, includeCurrentCost: includeCurrentCost ?? true, @@ -2272,6 +2364,10 @@ export class ClineProvider imageGenerationProvider, openRouterImageApiKey, openRouterImageGenerationSelectedModel, + memoryBackend, + agentMemoryUrl, + selfImprovingScope, + selfImprovingAutoSkillsScope, openAiCodexIsAuthenticated: await (async () => { try { const { openAiCodexOAuthManager } = await import("../../integrations/openai-codex/oauth") @@ -2468,6 +2564,10 @@ export class ClineProvider imageGenerationProvider: stateValues.imageGenerationProvider, openRouterImageApiKey: stateValues.openRouterImageApiKey, openRouterImageGenerationSelectedModel: stateValues.openRouterImageGenerationSelectedModel, + memoryBackend: stateValues.memoryBackend, + agentMemoryUrl: stateValues.agentMemoryUrl, + selfImprovingScope: stateValues.selfImprovingScope, + selfImprovingAutoSkillsScope: stateValues.selfImprovingAutoSkillsScope, } } @@ -2646,6 +2746,10 @@ export class ClineProvider return this.skillsManager } + public getSelfImprovingManager(): SelfImprovingManager { + return this.selfImprovingManager + } + /** * Check if the current state is compliant with MDM policy * @returns true if compliant or no MDM policy exists, false if MDM policy exists and user is non-compliant diff --git a/src/core/webview/__tests__/ClineProvider.apiHandlerRebuild.spec.ts b/src/core/webview/__tests__/ClineProvider.apiHandlerRebuild.spec.ts index 9d81880d4a..7b2909273b 100644 --- a/src/core/webview/__tests__/ClineProvider.apiHandlerRebuild.spec.ts +++ b/src/core/webview/__tests__/ClineProvider.apiHandlerRebuild.spec.ts @@ -9,6 +9,8 @@ import { ContextProxy } from "../../config/ContextProxy" import { Task, TaskOptions } from "../../task/Task" import { ClineProvider } from "../ClineProvider" +const mockWorkspacePath = vi.hoisted(() => ({ value: "/test/workspace-one" })) + // Mock setup vi.mock("fs/promises", () => ({ mkdir: vi.fn().mockResolvedValue(undefined), @@ -24,6 +26,10 @@ vi.mock("../../../utils/storage", () => ({ getGlobalStoragePath: vi.fn().mockResolvedValue("/test/storage/path"), })) +vi.mock("../../../utils/path", () => ({ + getWorkspacePath: vi.fn(() => mockWorkspacePath.value), +})) + vi.mock("p-wait-for", () => ({ __esModule: true, default: vi.fn().mockResolvedValue(undefined), @@ -143,6 +149,7 @@ describe("ClineProvider - API Handler Rebuild Guard", () => { beforeEach(async () => { vi.clearAllMocks() + mockWorkspacePath.value = "/test/workspace-one" if (!TelemetryService.hasInstance()) { TelemetryService.createInstance([]) @@ -579,4 +586,29 @@ describe("ClineProvider - API Handler Rebuild Guard", () => { expect(getModelId({})).toBeUndefined() }) }) + + test("includes self-improving scope and memory backend settings in provider state", async () => { + await (provider as any).setValue("selfImprovingScope", "workspace") + await (provider as any).setValue("selfImprovingAutoSkillsScope", "global") + await (provider as any).setValue("memoryBackend", "agentmemory") + await (provider as any).setValue("agentMemoryUrl", "http://agentmemory.internal:4001") + + const state = await provider.getState() + + expect((state as any).selfImprovingScope).toBe("workspace") + expect((state as any).selfImprovingAutoSkillsScope).toBe("global") + expect((state as any).memoryBackend).toBe("agentmemory") + expect((state as any).agentMemoryUrl).toBe("http://agentmemory.internal:4001") + }) + + test("refreshWorkspace reconfigures self-improving when the workspace path changes", async () => { + const onSettingsChanged = vi.fn().mockResolvedValue(undefined) + ;(provider as any).selfImprovingManager.onSettingsChanged = onSettingsChanged + await (provider as any).setValue("selfImprovingScope", "workspace") + mockWorkspacePath.value = "/test/workspace-two" + + await provider.refreshWorkspace() + + expect(onSettingsChanged).toHaveBeenCalledTimes(1) + }) }) diff --git a/src/core/webview/__tests__/webviewMessageHandler.spec.ts b/src/core/webview/__tests__/webviewMessageHandler.spec.ts index 17e0caebb0..9b45b3fa70 100644 --- a/src/core/webview/__tests__/webviewMessageHandler.spec.ts +++ b/src/core/webview/__tests__/webviewMessageHandler.spec.ts @@ -61,6 +61,9 @@ const mockFetchOpenAiCodexRateLimitInfo = vi.mocked(fetchOpenAiCodexRateLimitInf const mockClineProvider = { getState: vi.fn(), postMessageToWebview: vi.fn(), + selfImprovingManager: { + onSettingsChanged: vi.fn(), + }, customModesManager: { getCustomModes: vi.fn(), deleteCustomMode: vi.fn(), @@ -76,6 +79,7 @@ const mockClineProvider = { }, setValue: vi.fn(), getValue: vi.fn(), + getGlobalState: vi.fn(), }, log: vi.fn(), postStateToWebview: vi.fn(), @@ -860,6 +864,35 @@ describe("webviewMessageHandler - mcpEnabled", () => { }) }) +describe("webviewMessageHandler - self-improving memory settings", () => { + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(mockClineProvider.contextProxy.getGlobalState).mockReturnValue(undefined) + }) + + it("persists self-improving scope and memory backend settings and refreshes self-improving runtime", async () => { + await webviewMessageHandler(mockClineProvider, { + type: "updateSettings", + updatedSettings: { + selfImprovingScope: "workspace", + selfImprovingAutoSkillsScope: "global", + memoryBackend: "agentmemory", + agentMemoryUrl: "http://agentmemory.internal:4001", + } as any, + }) + + expect(mockClineProvider.contextProxy.setValue).toHaveBeenCalledWith("selfImprovingScope", "workspace") + expect(mockClineProvider.contextProxy.setValue).toHaveBeenCalledWith("selfImprovingAutoSkillsScope", "global") + expect(mockClineProvider.contextProxy.setValue).toHaveBeenCalledWith("memoryBackend", "agentmemory") + expect(mockClineProvider.contextProxy.setValue).toHaveBeenCalledWith( + "agentMemoryUrl", + "http://agentmemory.internal:4001", + ) + expect(mockClineProvider.selfImprovingManager.onSettingsChanged).toHaveBeenCalledWith(undefined) + expect(mockClineProvider.postStateToWebview).toHaveBeenCalledTimes(1) + }) +}) + describe("webviewMessageHandler - requestCommands", () => { beforeEach(() => { vi.clearAllMocks() diff --git a/src/core/webview/generateSystemPrompt.ts b/src/core/webview/generateSystemPrompt.ts index 8af2f5ff5d..433563e672 100644 --- a/src/core/webview/generateSystemPrompt.ts +++ b/src/core/webview/generateSystemPrompt.ts @@ -64,6 +64,7 @@ export const generateSystemPrompt = async (provider: ClineProvider, message: Web undefined, // todoList undefined, // modelId provider.getSkillsManager(), + provider.getSelfImprovingManager(), ) return systemPrompt diff --git a/src/core/webview/webviewMessageHandler.ts b/src/core/webview/webviewMessageHandler.ts index 429de051b8..bac44740c9 100644 --- a/src/core/webview/webviewMessageHandler.ts +++ b/src/core/webview/webviewMessageHandler.ts @@ -662,6 +662,9 @@ export const webviewMessageHandler = async ( case "updateSettings": if (message.updatedSettings) { + let experimentsUpdated = false + let selfImprovingSettingsUpdated = false + for (const [key, value] of Object.entries(message.updatedSettings)) { let newValue = value @@ -740,10 +743,18 @@ export const webviewMessageHandler = async ( continue } + experimentsUpdated = true newValue = { ...(getGlobalState("experiments") ?? experimentDefault), ...(value as Record), } + } else if ( + key === "memoryBackend" || + key === "agentMemoryUrl" || + key === "selfImprovingScope" || + key === "selfImprovingAutoSkillsScope" + ) { + selfImprovingSettingsUpdated = true } else if (key === "customSupportPrompts") { if (!value) { continue @@ -753,6 +764,12 @@ export const webviewMessageHandler = async ( await provider.contextProxy.setValue(key as keyof RooCodeSettings, newValue) } + if (experimentsUpdated || selfImprovingSettingsUpdated) { + await provider.selfImprovingManager.onSettingsChanged( + provider.contextProxy.getGlobalState("experiments"), + ) + } + await provider.postStateToWebview() } diff --git a/src/extension.ts b/src/extension.ts index 44c1243528..246f31a376 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -198,6 +198,13 @@ export async function activate(context: vscode.ExtensionContext) { // Initialize the provider *before* the Roo Code Cloud service. const provider = new ClineProvider(context, outputChannel, "sidebar", contextProxy, mdmService) + // Initialize self-improving manager (experiment-gated, zero overhead when disabled) + provider.initializeSelfImproving().catch((error) => { + outputChannel.appendLine( + `[SelfImproving] Initialization error: ${error instanceof Error ? error.message : String(error)}`, + ) + }) + // Initialize Roo Code Cloud service. const postStateListener = () => ClineProvider.getVisibleInstance()?.postStateToWebviewWithoutClineMessages() diff --git a/src/i18n/locales/en/skills.json b/src/i18n/locales/en/skills.json index 307b59d365..447947e4af 100644 --- a/src/i18n/locales/en/skills.json +++ b/src/i18n/locales/en/skills.json @@ -3,6 +3,7 @@ "name_length": "Skill name must be 1-{{maxLength}} characters (got {{length}})", "name_format": "Skill name must be lowercase letters/numbers/hyphens only (no leading/trailing hyphen, no consecutive hyphens)", "description_length": "Skill description must be 1-1024 characters (got {{length}})", + "invalid_structure": "Invalid SKILL.md structure: {{reason}}", "no_workspace": "Cannot create project skill: no workspace folder is open", "already_exists": "Skill \"{{name}}\" already exists at {{path}}", "not_found": "Skill \"{{name}}\" not found in {{source}}{{modeInfo}}", diff --git a/src/services/self-improving/ActionExecutor.ts b/src/services/self-improving/ActionExecutor.ts new file mode 100644 index 0000000000..8e0019a1ab --- /dev/null +++ b/src/services/self-improving/ActionExecutor.ts @@ -0,0 +1,251 @@ +import crypto from "crypto" + +import type { MemoryBackend } from "./MemoryBackend" +import type { SkillProvenance, SkillUsageStore } from "./SkillUsageStore" +import type { ImprovementAction, Logger } from "./types" + +interface SkillMutationManager { + createSkillFromContent( + name: string, + source: "global" | "project", + description: string, + content: string, + modeSlugs?: string[], + ): Promise + updateSkillContent(name: string, source: "global" | "project", content: string, mode?: string): Promise +} + +/** + * ActionExecutor - consumes the pending action queue and executes + * improvement actions transactionally. + * + * Each action type maps to a specific executor: + * - PROMPT_ENRICHMENT: writes to MemoryStore (environment) + * - ERROR_AVOIDANCE: writes to MemoryStore (environment, with error tags) + * - TOOL_PREFERENCE: writes to MemoryStore (environment, with tool tags) + * - SKILL_SUGGESTION: records in SkillUsageStore for future user approval + * - SKILL_CREATE / SKILL_UPDATE: safely mutate agent-managed skills via SkillsManager + * + * Actions are removed from the queue only after successful execution. + * Failed actions remain pending for later retry. + */ +export class ActionExecutor { + private readonly memoryStore: MemoryBackend + private readonly skillUsageStore: SkillUsageStore + private readonly logger: Logger + private readonly skillsManager?: SkillMutationManager + + constructor( + memoryStore: MemoryBackend, + skillUsageStore: SkillUsageStore, + logger: Logger, + skillsManager?: SkillMutationManager, + ) { + this.memoryStore = memoryStore + this.skillUsageStore = skillUsageStore + this.logger = logger + this.skillsManager = skillsManager + } + + /** + * Execute a single improvement action. + * Returns true if the action was executed successfully. + */ + async execute(action: ImprovementAction): Promise { + try { + let executed = false + + switch (action.actionType) { + case "PROMPT_ENRICHMENT": + executed = await this.executePromptEnrichment(action) + break + case "ERROR_AVOIDANCE": + executed = await this.executeErrorAvoidance(action) + break + case "TOOL_PREFERENCE": + executed = await this.executeToolPreference(action) + break + case "SKILL_SUGGESTION": + executed = await this.executeSkillSuggestion(action) + break + case "SKILL_CREATE": + executed = await this.executeSkillCreate(action) + break + case "SKILL_UPDATE": + executed = await this.executeSkillUpdate(action) + break + default: + this.logger.appendLine(`[ActionExecutor] Unknown action type: ${action.actionType}`) + return false + } + + this.logger.appendLine( + `[ActionExecutor] ${executed ? "Executed" : "Deferred"} ${action.actionType} action ${action.id}`, + ) + + return executed + } catch (error) { + this.logger.appendLine( + `[ActionExecutor] Execution error for ${action.id}: ${error instanceof Error ? error.message : String(error)}`, + ) + return false + } + } + + /** + * Execute a batch of actions. + * Returns the set of successfully executed action IDs. + */ + async executeBatch(actions: ImprovementAction[]): Promise> { + const succeeded = new Set() + + for (const action of actions) { + const ok = await this.execute(action) + if (ok) { + succeeded.add(action.id) + } + } + + return succeeded + } + + private async executePromptEnrichment(action: ImprovementAction): Promise { + const summary = this.readStringPayload(action.payload.summary) + if (!summary) { + return false + } + + await this.memoryStore.store({ + content: summary, + source: "learning", + tags: ["learned", "prompt"], + }) + + return true + } + + private async executeErrorAvoidance(action: ImprovementAction): Promise { + const summary = this.readStringPayload(action.payload.summary) + const errorKeys = this.readStringArrayPayload(action.payload.errorKeys) + + if (!summary) { + return false + } + + await this.memoryStore.store({ + content: summary, + source: "learning", + tags: ["error-avoidance", ...errorKeys.map((key) => `error:${key}`)], + }) + + return true + } + + private async executeToolPreference(action: ImprovementAction): Promise { + const summary = this.readStringPayload(action.payload.summary) + const toolNames = this.readStringArrayPayload(action.payload.toolNames) + + if (!summary) { + return false + } + + await this.memoryStore.store({ + content: summary, + source: "learning", + tags: ["tool-preference", ...toolNames.map((toolName) => `tool:${toolName}`)], + }) + + return true + } + + private async executeSkillSuggestion(action: ImprovementAction): Promise { + const summary = this.readStringPayload(action.payload.summary) + if (!summary) { + return false + } + + const skillName = this.readStringPayload(action.payload.skillName) ?? summary + const skillId = + this.readStringPayload(action.payload.skillId) ?? + `suggested:${crypto.createHash("sha256").update(skillName.toLowerCase()).digest("hex").slice(0, 16)}` + const createdBy = this.readSkillProvenance(action.payload.createdBy) ?? "agent" + + this.skillUsageStore.getOrCreate(skillId, skillName, createdBy) + this.logger.appendLine(`[ActionExecutor] Skill suggestion recorded: ${summary}`) + + return true + } + + private async executeSkillCreate(action: ImprovementAction): Promise { + if (!this.skillsManager) { + return false + } + + const skillName = this.readStringPayload(action.payload.skillName) + const description = this.readStringPayload(action.payload.description) + const content = this.readStringPayload(action.payload.content) + const source = this.readSkillSource(action.payload.source) + const modeSlugs = this.readStringArrayPayload(action.payload.modeSlugs) + const skillId = this.readStringPayload(action.payload.skillId) ?? this.buildSkillId(skillName, source) + const createdBy = this.readSkillProvenance(action.payload.createdBy) ?? "agent" + + if (!skillName || !description || !content || !source || !skillId) { + return false + } + + await this.skillsManager.createSkillFromContent(skillName, source, description, content, modeSlugs) + this.skillUsageStore.getOrCreate(skillId, skillName, createdBy) + this.logger.appendLine(`[ActionExecutor] Skill created: ${skillName}`) + return true + } + + private async executeSkillUpdate(action: ImprovementAction): Promise { + if (!this.skillsManager) { + return false + } + + const skillName = this.readStringPayload(action.payload.skillName) + const content = this.readStringPayload(action.payload.content) + const source = this.readSkillSource(action.payload.source) + const mode = this.readStringPayload(action.payload.mode) + const skillId = this.readStringPayload(action.payload.skillId) ?? this.buildSkillId(skillName, source) + + if (!skillName || !content || !source || !skillId) { + return false + } + + await this.skillsManager.updateSkillContent(skillName, source, content, mode) + this.skillUsageStore.getOrCreate(skillId, skillName, "agent") + await this.skillUsageStore.bumpPatch(skillId) + this.logger.appendLine(`[ActionExecutor] Skill updated: ${skillName}`) + return true + } + + private readStringPayload(value: unknown): string | undefined { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined + } + + private readStringArrayPayload(value: unknown): string[] { + if (!Array.isArray(value)) { + return [] + } + + return Array.from( + new Set(value.filter((item): item is string => typeof item === "string" && item.trim().length > 0)), + ) + } + + private readSkillProvenance(value: unknown): SkillProvenance | undefined { + return value === "agent" || value === "user" || value === "bundled" || value === "hub" || value === "unknown" + ? value + : undefined + } + + private readSkillSource(value: unknown): "global" | "project" | undefined { + return value === "global" || value === "project" ? value : undefined + } + + private buildSkillId(skillName: string | undefined, source: "global" | "project" | undefined): string | undefined { + return skillName && source ? `skill:${source}:${skillName}` : undefined + } +} diff --git a/src/services/self-improving/AgentMemoryAdapter.ts b/src/services/self-improving/AgentMemoryAdapter.ts new file mode 100644 index 0000000000..e042fa5039 --- /dev/null +++ b/src/services/self-improving/AgentMemoryAdapter.ts @@ -0,0 +1,257 @@ +import type { MemoryEntry } from "@roo-code/types" + +import type { MemoryBackend, MemoryBackendType } from "./MemoryBackend" +import type { Logger } from "./types" + +/** + * Default agentmemory server URL + */ +const DEFAULT_AGENTMEMORY_URL = "http://localhost:3111" + +type AgentMemoryApiResult = { + id: string + content: string + metadata?: Record +} + +/** + * AgentMemoryAdapter — implements MemoryBackend via agentmemory REST API. + * + * agentmemory (https://github.com/rohitg00/agentmemory) is a service-first + * memory system. This adapter connects to its REST API when the server is + * running, and gracefully degrades to no-op when it's not. + * + * Key REST endpoints used: + * POST /agentmemory/observe — store an observation + * POST /agentmemory/search — semantic search + * POST /agentmemory/remember — recall recent memories + * POST /agentmemory/forget — remove a memory + * GET /agentmemory/livez — health check + */ +export class AgentMemoryAdapter implements MemoryBackend { + private readonly baseUrl: string + private readonly logger: Logger + private available = false + private healthCheckInterval: ReturnType | null = null + private initialized = false + + constructor(logger: Logger, baseUrl?: string) { + this.baseUrl = baseUrl || DEFAULT_AGENTMEMORY_URL + this.logger = logger + } + + get backendType(): MemoryBackendType { + return "agentmemory" + } + + /** + * Initialize the adapter — check if agentmemory server is available. + */ + async initialize(): Promise { + if (this.initialized) return + + this.available = await this.checkHealth() + + if (this.available) { + this.logger.appendLine(`[AgentMemoryAdapter] Connected to agentmemory at ${this.baseUrl}`) + } else { + this.logger.appendLine( + `[AgentMemoryAdapter] agentmemory server not available at ${this.baseUrl} — will degrade gracefully`, + ) + } + + this.healthCheckInterval = setInterval(async () => { + this.available = await this.checkHealth() + }, 30000) + + this.initialized = true + } + + /** + * Check if agentmemory server is healthy. + */ + private async checkHealth(): Promise { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 2000) + + try { + const response = await fetch(`${this.baseUrl}/agentmemory/livez`, { + signal: controller.signal, + }) + + return response.ok + } catch { + return false + } finally { + clearTimeout(timeout) + } + } + + /** + * Make a POST request to agentmemory API. + */ + private async post(path: string, body: unknown): Promise { + if (!this.available) return null + + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 5000) + + try { + const response = await fetch(`${this.baseUrl}${path}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + signal: controller.signal, + }) + + if (!response.ok) { + this.logger.appendLine(`[AgentMemoryAdapter] POST ${path} failed: ${response.status}`) + return null + } + + return (await response.json()) as T + } catch (error) { + this.logger.appendLine( + `[AgentMemoryAdapter] POST ${path} error: ${error instanceof Error ? error.message : String(error)}`, + ) + this.available = false + return null + } finally { + clearTimeout(timeout) + } + } + + /** + * Store a memory entry via agentmemory observe endpoint. + */ + async store(entry: Omit): Promise { + const result = await this.post<{ id: string }>("/agentmemory/observe", { + content: entry.content, + metadata: { + source: entry.source, + tags: entry.tags, + relevanceScore: entry.relevanceScore, + expiresAt: entry.expiresAt, + }, + }) + + if (!result) return null + + return { + id: result.id, + content: entry.content, + source: entry.source, + createdAt: Date.now(), + updatedAt: Date.now(), + relevanceScore: entry.relevanceScore, + tags: entry.tags, + expiresAt: entry.expiresAt, + } + } + + /** + * Search memory entries via agentmemory search endpoint. + */ + async search(query: string, maxResults: number = 10): Promise { + const result = await this.post<{ results: AgentMemoryApiResult[] }>("/agentmemory/search", { + query, + limit: maxResults, + }) + + if (!result?.results) return [] + + return result.results.map((entry) => this.mapResultToMemoryEntry(entry)) + } + + /** + * Recall recent memory entries via agentmemory remember endpoint. + */ + async recall(maxResults: number = 20): Promise { + const result = await this.post<{ memories: AgentMemoryApiResult[] }>("/agentmemory/remember", { + limit: maxResults, + }) + + if (!result?.memories) return [] + + return result.memories.map((entry) => this.mapResultToMemoryEntry(entry)) + } + + /** + * Remove a memory entry by ID via agentmemory forget endpoint. + */ + async forget(id: string): Promise { + const result = await this.post<{ success: boolean }>("/agentmemory/forget", { id }) + return result?.success === true + } + + /** + * Remove entries matching content substring. + * Uses agentmemory search + forget pattern. + */ + async forgetByContent(substring: string): Promise { + const normalized = substring.trim().toLowerCase() + if (!normalized) { + return 0 + } + + const entries = await this.search(substring.trim(), 50) + let removed = 0 + + for (const entry of entries) { + if (entry.content.toLowerCase().includes(normalized)) { + const ok = await this.forget(entry.id) + if (ok) removed += 1 + } + } + + return removed + } + + /** + * Get backend statistics. + */ + async getStats(): Promise<{ entryCount: number; backend: string }> { + if (!this.available) { + return { entryCount: 0, backend: "agentmemory (unavailable)" } + } + + const memories = await this.recall(1000) + return { + entryCount: memories.length, + backend: "agentmemory", + } + } + + /** + * Clear all entries via agentmemory governance delete. + */ + async clear(): Promise { + await this.post("/agentmemory/governance/bulk-delete", { all: true }) + } + + /** + * Dispose the adapter — stop health check interval. + */ + async dispose(): Promise { + if (this.healthCheckInterval) { + clearInterval(this.healthCheckInterval) + this.healthCheckInterval = null + } + + this.available = false + this.initialized = false + } + + private mapResultToMemoryEntry(entry: AgentMemoryApiResult): MemoryEntry { + return { + id: entry.id, + content: entry.content, + source: (entry.metadata?.source as MemoryEntry["source"]) || "learning", + createdAt: (entry.metadata?.createdAt as number) || Date.now(), + updatedAt: (entry.metadata?.updatedAt as number) || Date.now(), + relevanceScore: entry.metadata?.relevanceScore as number | undefined, + tags: entry.metadata?.tags as string[] | undefined, + expiresAt: entry.metadata?.expiresAt as number | undefined, + } + } +} diff --git a/src/services/self-improving/CodeIndexAdapter.ts b/src/services/self-improving/CodeIndexAdapter.ts new file mode 100644 index 0000000000..937fcc5425 --- /dev/null +++ b/src/services/self-improving/CodeIndexAdapter.ts @@ -0,0 +1,44 @@ +import type { CodeIndexInfo, Logger } from "./types" + +/** + * CodeIndexAdapter - thin read-only adapter for code index integration. + * + * When code index is unavailable or disabled, returns a no-op payload + * and the learning loop proceeds normally. + */ +export class CodeIndexAdapter { + private readonly getCodeIndexInfo: (() => CodeIndexInfo) | undefined + + constructor( + private readonly logger?: Logger, + getCodeIndexInfo?: () => CodeIndexInfo, + ) { + this.getCodeIndexInfo = getCodeIndexInfo + } + + /** + * Get current code index info. + * Returns a safe default if the adapter is not configured. + */ + getInfo(): CodeIndexInfo { + if (!this.getCodeIndexInfo) { + return { available: false, hits: 0 } + } + + try { + return this.getCodeIndexInfo() + } catch (error) { + this.logger?.appendLine( + `[CodeIndexAdapter] Error getting code index info: ${error instanceof Error ? error.message : String(error)}`, + ) + return { available: false, hits: 0 } + } + } + + /** + * Check if code index is available and configured. + */ + isAvailable(): boolean { + return this.getInfo().available + } +} diff --git a/src/services/self-improving/CuratorService.ts b/src/services/self-improving/CuratorService.ts new file mode 100644 index 0000000000..76add6e91b --- /dev/null +++ b/src/services/self-improving/CuratorService.ts @@ -0,0 +1,423 @@ +import * as fs from "fs/promises" +import * as path from "path" +import crypto from "crypto" + +import { safeWriteJson } from "../../utils/safeWriteJson" +import type { Logger } from "./types" +import type { SkillTelemetryRecord, SkillUsageStore } from "./SkillUsageStore" + +/** + * Curator configuration + */ +export interface CuratorConfig { + /** Minimum interval between curator runs (ms) */ + intervalMs: number + /** Minimum idle time since last user activity before curator runs (ms) */ + minIdleMs: number + /** Whether to defer the first curator run */ + firstRunDeferred: boolean + /** Days of inactivity before a skill is marked stale */ + staleAfterDays: number + /** Days of inactivity before a stale skill is archived */ + archiveAfterDays: number + /** Whether to create pre-run backups */ + backupsEnabled: boolean + /** Maximum number of backup snapshots to retain */ + maxBackups: number +} + +/** + * Default curator configuration + */ +export const DEFAULT_CURATOR_CONFIG: CuratorConfig = { + intervalMs: 3_600_000, + minIdleMs: 300_000, + firstRunDeferred: true, + staleAfterDays: 14, + archiveAfterDays: 60, + backupsEnabled: true, + maxBackups: 5, +} + +/** + * Curator run report + */ +export interface CuratorReport { + runId: string + timestamp: number + durationMs: number + transitions: Array<{ + skillId: string + skillName: string + fromState: string + toState: string + reason: string + }> + stats: { + totalSkills: number + activeSkills: number + staleSkills: number + archivedSkills: number + pinnedSkills: number + transitionsApplied: number + } + backupPath?: string + error?: string +} + +type CuratorStatus = { + lastRunAt: number + firstRunDone: boolean + config: CuratorConfig +} + +/** + * CuratorService — telemetry-driven skill lifecycle management. + */ +export class CuratorService { + private readonly baseDir: string + private readonly statePath: string + private readonly backupsDir: string + private readonly reportsDir: string + private readonly skillUsageStore: SkillUsageStore + private readonly logger: Logger + private config: CuratorConfig + private lastRunAt = 0 + private firstRunDone = false + private initialized = false + + constructor(baseDir: string, skillUsageStore: SkillUsageStore, logger: Logger, config?: Partial) { + this.baseDir = path.join(baseDir, "self-improving", "curator") + this.statePath = path.join(this.baseDir, "state.json") + this.backupsDir = path.join(this.baseDir, "backups") + this.reportsDir = path.join(this.baseDir, "reports") + this.skillUsageStore = skillUsageStore + this.logger = logger + this.config = { ...DEFAULT_CURATOR_CONFIG, ...config } + } + + async initialize(): Promise { + if (this.initialized) { + return + } + + try { + await fs.mkdir(this.backupsDir, { recursive: true }) + await fs.mkdir(this.reportsDir, { recursive: true }) + await this.loadState() + this.logger.appendLine("[CuratorService] Initialized") + } catch (error) { + this.logger.appendLine( + `[CuratorService] Initialization error: ${error instanceof Error ? error.message : String(error)}`, + ) + } finally { + this.initialized = true + } + } + + shouldRun(now: number, lastUserActivityAt?: number): boolean { + if (this.config.firstRunDeferred && !this.firstRunDone && this.lastRunAt === 0) { + return false + } + + if (now - this.lastRunAt < this.config.intervalMs) { + return false + } + + if (typeof lastUserActivityAt === "number" && now - lastUserActivityAt < this.config.minIdleMs) { + return false + } + + return true + } + + async run(now: number, lastUserActivityAt?: number): Promise { + await this.initialize() + + const startedAt = Date.now() + const runId = crypto.randomUUID() + const report: CuratorReport = { + runId, + timestamp: now, + durationMs: 0, + transitions: [], + stats: { + totalSkills: 0, + activeSkills: 0, + staleSkills: 0, + archivedSkills: 0, + pinnedSkills: 0, + transitionsApplied: 0, + }, + } + + try { + if (this.shouldDeferFirstRun()) { + this.firstRunDone = true + await this.saveState() + report.error = "Skipped: first-run deferral" + report.durationMs = Date.now() - startedAt + await this.writeReport(report) + return report + } + + if (!this.shouldRun(now, lastUserActivityAt)) { + report.error = "Skipped: gates not satisfied" + report.durationMs = Date.now() - startedAt + await this.writeReport(report) + return report + } + + // Set lastRunAt immediately to prevent concurrent runs + this.lastRunAt = now + + if (this.config.backupsEnabled) { + report.backupPath = await this.createBackup(runId) + } + + this.assignStats(report) + report.transitions = await this.applyDeterministicTransitions() + await this.runCuratorReview(report) + report.stats.transitionsApplied = report.transitions.length + this.assignStats(report) + + this.firstRunDone = true + await this.saveState() + + report.durationMs = Date.now() - startedAt + await this.writeReport(report) + this.logger.appendLine( + `[CuratorService] Run ${runId}: ${report.transitions.length} transitions in ${report.durationMs}ms`, + ) + } catch (error) { + report.error = error instanceof Error ? error.message : String(error) + report.durationMs = Date.now() - startedAt + this.logger.appendLine(`[CuratorService] Run error: ${report.error}`) + await this.writeReport(report) + } + + return report + } + + async getLatestReport(): Promise { + try { + const entries = await fs.readdir(this.reportsDir, { withFileTypes: true }) + const candidates = await Promise.all( + entries + .filter((entry) => entry.isDirectory()) + .map(async (entry) => { + const runPath = path.join(this.reportsDir, entry.name, "run.json") + const stats = await fs.stat(runPath) + return { runPath, mtimeMs: stats.mtimeMs } + }), + ) + + if (candidates.length === 0) { + return null + } + + candidates.sort((left, right) => right.mtimeMs - left.mtimeMs) + const raw = await fs.readFile(candidates[0].runPath, "utf-8") + return JSON.parse(raw) as CuratorReport + } catch { + return null + } + } + + getConfig(): Readonly { + return this.config + } + + setConfig(config: Partial): void { + this.config = { ...this.config, ...config } + } + + getStatus(): CuratorStatus { + return { + lastRunAt: this.lastRunAt, + firstRunDone: this.firstRunDone, + config: { ...this.config }, + } + } + + private async loadState(): Promise { + try { + const raw = await fs.readFile(this.statePath, "utf-8") + const parsed = JSON.parse(raw) as Partial + this.lastRunAt = typeof parsed.lastRunAt === "number" ? parsed.lastRunAt : 0 + this.firstRunDone = parsed.firstRunDone === true + } catch { + this.lastRunAt = 0 + this.firstRunDone = false + } + } + + private async saveState(): Promise { + try { + await safeWriteJson( + this.statePath, + { + lastRunAt: this.lastRunAt, + firstRunDone: this.firstRunDone, + }, + { prettyPrint: true }, + ) + } catch (error) { + this.logger.appendLine( + `[CuratorService] Save state error: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + + private shouldDeferFirstRun(): boolean { + return this.config.firstRunDeferred && !this.firstRunDone && this.lastRunAt === 0 + } + + private async createBackup(runId: string): Promise { + const backupDir = path.join(this.backupsDir, `backup-${Date.now()}-${runId}`) + await fs.mkdir(backupDir, { recursive: true }) + await safeWriteJson( + path.join(backupDir, "snapshot.json"), + { + createdAt: Date.now(), + curatorState: { + lastRunAt: this.lastRunAt, + firstRunDone: this.firstRunDone, + }, + skillUsage: this.skillUsageStore.getAll(), + }, + { prettyPrint: true }, + ) + await this.cleanupOldBackups() + return backupDir + } + + private async cleanupOldBackups(): Promise { + try { + const entries = await fs.readdir(this.backupsDir, { withFileTypes: true }) + const backups = await Promise.all( + entries + .filter((entry) => entry.isDirectory() && entry.name.startsWith("backup-")) + .map(async (entry) => { + const backupPath = path.join(this.backupsDir, entry.name) + const stats = await fs.stat(backupPath) + return { backupPath, mtimeMs: stats.mtimeMs } + }), + ) + + backups.sort((left, right) => right.mtimeMs - left.mtimeMs) + for (const staleBackup of backups.slice(this.config.maxBackups)) { + await fs.rm(staleBackup.backupPath, { recursive: true, force: true }) + } + } catch { + // Best-effort retention cleanup. + } + } + + private assignStats(report: CuratorReport): void { + const stats = this.skillUsageStore.getStats() + report.stats.totalSkills = stats.total + report.stats.activeSkills = stats.active + report.stats.staleSkills = stats.stale + report.stats.archivedSkills = stats.archived + report.stats.pinnedSkills = stats.pinned + } + + private async applyDeterministicTransitions(): Promise { + const transitions: CuratorReport["transitions"] = [] + + for (const candidate of this.skillUsageStore.getStaleCandidates(this.config.staleAfterDays)) { + if (this.isProtected(candidate)) { + continue + } + + await this.skillUsageStore.transitionState(candidate.skillId, "stale") + transitions.push({ + skillId: candidate.skillId, + skillName: candidate.skillName, + fromState: "active", + toState: "stale", + reason: `No activity for ${this.config.staleAfterDays} days`, + }) + } + + for (const candidate of this.skillUsageStore.getArchiveCandidates(this.config.archiveAfterDays)) { + if (this.isProtected(candidate)) { + continue + } + + await this.skillUsageStore.transitionState(candidate.skillId, "archived") + transitions.push({ + skillId: candidate.skillId, + skillName: candidate.skillName, + fromState: "stale", + toState: "archived", + reason: `No activity for ${this.config.archiveAfterDays} days`, + }) + } + + return transitions + } + + private isProtected(record: SkillTelemetryRecord): boolean { + return record.pinned || record.createdBy !== "agent" + } + + private async runCuratorReview(_report: CuratorReport): Promise { + // Reserved for future rubric-driven LLM curator review. + } + + private async writeReport(report: CuratorReport): Promise { + try { + const runDir = path.join(this.reportsDir, report.runId) + await fs.mkdir(runDir, { recursive: true }) + await safeWriteJson(path.join(runDir, "run.json"), report, { prettyPrint: true }) + await fs.writeFile(path.join(runDir, "REPORT.md"), this.buildReportMarkdown(report), "utf-8") + } catch (error) { + this.logger.appendLine( + `[CuratorService] Report write error: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + + private buildReportMarkdown(report: CuratorReport): string { + const lines = [ + `# Curator Run Report: ${report.runId}`, + "", + `**Timestamp**: ${new Date(report.timestamp).toISOString()}`, + `**Duration**: ${report.durationMs}ms`, + "", + "## Summary", + "", + "| Metric | Value |", + "|--------|-------|", + `| Total Skills | ${report.stats.totalSkills} |`, + `| Active | ${report.stats.activeSkills} |`, + `| Stale | ${report.stats.staleSkills} |`, + `| Archived | ${report.stats.archivedSkills} |`, + `| Pinned | ${report.stats.pinnedSkills} |`, + `| Transitions Applied | ${report.stats.transitionsApplied} |`, + "", + ] + + if (report.transitions.length > 0) { + lines.push("## Transitions", "", "| Skill | From | To | Reason |", "|-------|------|----|--------|") + for (const transition of report.transitions) { + lines.push( + `| ${transition.skillName} | ${transition.fromState} | ${transition.toState} | ${transition.reason} |`, + ) + } + lines.push("") + } + + if (report.backupPath) { + lines.push(`**Backup**: ${report.backupPath}`, "") + } + + if (report.error) { + lines.push("## Error", "", report.error, "") + } + + return lines.join("\n") + } +} diff --git a/src/services/self-improving/FeedbackCollector.ts b/src/services/self-improving/FeedbackCollector.ts new file mode 100644 index 0000000000..af19168a63 --- /dev/null +++ b/src/services/self-improving/FeedbackCollector.ts @@ -0,0 +1,122 @@ +import crypto from "crypto" + +import type { CodeIndexInfo, FeedbackSignal, LearningEvent, TaskEventInfo } from "./types" + +/** + * FeedbackCollector - normalizes task/user/tool/code-index signals + * into structured LearningEvent objects. + * + * This is a stateless converter - it creates events from raw signals + * without side effects. The caller (SelfImprovingManager) owns + * persistence and lifecycle. + */ +export class FeedbackCollector { + /** + * Create a learning event from a task completion signal. + */ + createTaskEvent(info: TaskEventInfo): LearningEvent { + const signal: FeedbackSignal = info.success ? "TASK_SUCCESS" : "TASK_FAILURE" + + return { + id: crypto.randomUUID(), + signal, + timestamp: Date.now(), + taskId: info.taskId, + workspacePath: info.workspacePath, + mode: info.mode, + context: { + userTurnCount: info.userTurnCount, + toolIterationCount: info.toolIterationCount, + toolNames: info.toolNames, + promptFingerprint: info.promptFingerprint, + errorKey: info.errorKey, + }, + outcome: { + success: info.success, + corrected: info.corrected, + confidenceDelta: info.success ? 0.05 : -0.1, + }, + } + } + + /** + * Create a learning event from a user correction signal. + */ + createCorrectionEvent(info: TaskEventInfo): LearningEvent { + return { + id: crypto.randomUUID(), + signal: "USER_CORRECTION", + timestamp: Date.now(), + taskId: info.taskId, + workspacePath: info.workspacePath, + mode: info.mode, + context: { + toolNames: info.toolNames, + errorKey: info.errorKey, + promptFingerprint: info.promptFingerprint, + }, + outcome: { + corrected: true, + confidenceDelta: -0.15, + }, + } + } + + /** + * Create a learning event from a pattern repeat signal. + */ + createPatternRepeatEvent(patternId: string, taskId?: string, mode?: string): LearningEvent { + return { + id: crypto.randomUUID(), + signal: "PATTERN_REPEAT", + timestamp: Date.now(), + taskId, + mode, + context: { + promptFingerprint: patternId, + }, + outcome: { + confidenceDelta: 0.02, + }, + } + } + + /** + * Create a learning event from a code index hit. + */ + createCodeIndexEvent(codeIndex: CodeIndexInfo, taskId?: string): LearningEvent { + return { + id: crypto.randomUUID(), + signal: "CODE_INDEX_HIT", + timestamp: Date.now(), + taskId, + context: { + codeIndex: { + available: codeIndex.available, + hits: codeIndex.hits, + topScore: codeIndex.topScore, + }, + }, + outcome: { + confidenceDelta: codeIndex.hits > 0 ? 0.03 : 0, + }, + } + } + + /** + * Create a learning event from a prompt quality signal. + */ + createPromptQualityEvent(quality: number, promptFingerprint?: string): LearningEvent { + return { + id: crypto.randomUUID(), + signal: "PROMPT_QUALITY", + timestamp: Date.now(), + context: { + promptFingerprint, + }, + outcome: { + confidenceDelta: quality > 0.5 ? 0.01 : -0.01, + }, + } + } +} diff --git a/src/services/self-improving/ImprovementApplier.ts b/src/services/self-improving/ImprovementApplier.ts new file mode 100644 index 0000000000..f15e041c38 --- /dev/null +++ b/src/services/self-improving/ImprovementApplier.ts @@ -0,0 +1,302 @@ +import crypto from "crypto" + +import type { SkillProvenance } from "./SkillUsageStore" +import type { ImprovementAction, LearnedPattern, PromptContext } from "./types" + +interface ImprovementApplierOptions { + getSkillNames?: () => string[] + getSkillProvenance?: (name: string) => SkillProvenance | string + getSkillProvenanceForSource?: (name: string, source: "global" | "project") => SkillProvenance | string + hasSkill?: (name: string, source: "global" | "project") => boolean + isAutoSkillsEnabled?: () => boolean + getAutoSkillsScope?: () => "workspace" | "global" +} + +/** + * ImprovementApplier - converts learned patterns into actionable improvements. + * + * Generates: + * - Prompt enrichment context (bounded, ordered by confidence) + * - Tool preference adjustments + * - Error avoidance hints + * - Skill suggestions / mutations for reusable workflows + */ +export class ImprovementApplier { + private readonly getSkillNames: () => string[] + private readonly getSkillProvenance: (name: string) => SkillProvenance | string + private readonly getSkillProvenanceForSource: ( + name: string, + source: "global" | "project", + ) => SkillProvenance | string + private readonly hasSkill: (name: string, source: "global" | "project") => boolean + private readonly isAutoSkillsEnabled: () => boolean + private readonly getAutoSkillsScope: () => "workspace" | "global" + + constructor(options: ImprovementApplierOptions = {}) { + this.getSkillNames = options.getSkillNames ?? (() => []) + this.getSkillProvenance = options.getSkillProvenance ?? (() => "unknown") + this.getSkillProvenanceForSource = + options.getSkillProvenanceForSource ?? ((name: string) => this.getSkillProvenance(name)) + this.hasSkill = + options.hasSkill ?? + ((name: string, source: "global" | "project") => + source === "project" && this.getSkillNames().includes(name)) + this.isAutoSkillsEnabled = options.isAutoSkillsEnabled ?? (() => false) + this.getAutoSkillsScope = options.getAutoSkillsScope ?? (() => "workspace") + } + + /** + * Generate prompt context from active patterns. + * Returns at most maxEntries entries, ordered by confidence descending. + */ + getPromptContext(patterns: LearnedPattern[], maxEntries = 5, revision = 0): PromptContext { + const activePatterns = patterns + .filter((pattern) => pattern.state === "active") + .sort((left, right) => right.confidenceScore - left.confidenceScore) + .slice(0, maxEntries) + + return { + entries: activePatterns.map((pattern) => ({ + type: pattern.patternType, + summary: pattern.summary, + confidence: pattern.confidenceScore, + })), + revision, + } + } + + /** + * Generate improvement actions from patterns. + */ + generateActions(patterns: LearnedPattern[]): ImprovementAction[] { + const actions: ImprovementAction[] = [] + const now = Date.now() + + for (const pattern of patterns) { + if (pattern.state !== "active") { + continue + } + + switch (pattern.patternType) { + case "error": + actions.push(this.createErrorAvoidanceAction(pattern, now)) + break + case "tool": + actions.push(this.createToolPreferenceAction(pattern, now)) + if (this.isAutoSkillsEnabled()) { + const skillAction = this.createSkillMutationAction(pattern, now) + if (skillAction) { + actions.push(skillAction) + } + } + break + case "prompt": + actions.push(this.createPromptEnrichmentAction(pattern, now)) + break + case "skill": + actions.push(this.createSkillSuggestionAction(pattern, now)) + break + } + } + + return actions + } + + /** + * Create an error avoidance action from an error pattern. + */ + private createErrorAvoidanceAction(pattern: LearnedPattern, now: number): ImprovementAction { + return { + id: crypto.randomUUID(), + actionType: "ERROR_AVOIDANCE", + target: "task-execution", + payload: { + patternId: pattern.id, + errorKeys: pattern.context.errorKeys, + summary: pattern.summary, + confidence: pattern.confidenceScore, + }, + timestamp: now, + } + } + + /** + * Create a tool preference action from a tool pattern. + */ + private createToolPreferenceAction(pattern: LearnedPattern, now: number): ImprovementAction { + return { + id: crypto.randomUUID(), + actionType: "TOOL_PREFERENCE", + target: "task-execution", + payload: { + patternId: pattern.id, + toolNames: pattern.context.toolNames, + summary: pattern.summary, + confidence: pattern.confidenceScore, + }, + timestamp: now, + } + } + + /** + * Create a prompt enrichment action from a prompt pattern. + */ + private createPromptEnrichmentAction(pattern: LearnedPattern, now: number): ImprovementAction { + return { + id: crypto.randomUUID(), + actionType: "PROMPT_ENRICHMENT", + target: "system-prompt", + payload: { + patternId: pattern.id, + summary: pattern.summary, + confidence: pattern.confidenceScore, + }, + timestamp: now, + } + } + + /** + * Create a skill suggestion action from a skill pattern. + */ + private createSkillSuggestionAction(pattern: LearnedPattern, now: number): ImprovementAction { + return { + id: crypto.randomUUID(), + actionType: "SKILL_SUGGESTION", + target: "skills-manager", + payload: { + patternId: pattern.id, + summary: pattern.summary, + confidence: pattern.confidenceScore, + toolNames: pattern.context.toolNames, + }, + timestamp: now, + } + } + + private createSkillMutationAction(pattern: LearnedPattern, now: number): ImprovementAction | undefined { + const toolNames = this.normalizeToolNames(pattern.context.toolNames) + if (toolNames.length < 2 || pattern.frequency < 3 || pattern.successRate < 0.75) { + return undefined + } + + const skillName = this.buildWorkflowSkillName(toolNames) + const summary = `Capture reusable workflow for ${toolNames.join(", ")}` + const description = `Use when tasks repeatedly succeed with ${toolNames.join(" and ")}.` + const content = this.buildSkillContent(skillName, description, toolNames) + const modeSlugs = + pattern.context.modes && pattern.context.modes.length > 0 ? [...new Set(pattern.context.modes)] : undefined + const source = this.getAutoSkillsScope() === "global" ? "global" : "project" + const skillExists = this.hasSkill(skillName, source) + const skillId = this.buildSkillId(skillName, source) + + if ( + skillExists && + this.normalizeSkillProvenance(this.getSkillProvenanceForSource(skillName, source)) === "agent" + ) { + return { + id: crypto.randomUUID(), + actionType: "SKILL_UPDATE", + target: "skills-manager", + payload: { + patternId: pattern.id, + skillId, + skillName, + summary, + description, + content, + source, + mode: modeSlugs?.[0], + modeSlugs, + createdBy: "agent", + confidence: pattern.confidenceScore, + toolNames, + }, + timestamp: now, + } + } + + if (!skillExists) { + return { + id: crypto.randomUUID(), + actionType: "SKILL_CREATE", + target: "skills-manager", + payload: { + patternId: pattern.id, + skillId, + skillName, + summary, + description, + content, + source, + modeSlugs, + createdBy: "agent", + confidence: pattern.confidenceScore, + toolNames, + }, + timestamp: now, + } + } + + return undefined + } + + private normalizeToolNames(toolNames: string[] | undefined): string[] { + if (!Array.isArray(toolNames)) { + return [] + } + + return Array.from( + new Set(toolNames.map((toolName) => toolName.trim()).filter((toolName) => toolName.length > 0)), + ).sort() + } + + private normalizeSkillProvenance(value: SkillProvenance | string): SkillProvenance { + return value === "agent" || value === "user" || value === "bundled" || value === "hub" ? value : "unknown" + } + + private buildWorkflowSkillName(toolNames: string[]): string { + return `workflow-${toolNames + .map((toolName) => + toolName + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""), + ) + .join("-")}` + } + + private buildSkillId(skillName: string, source: "global" | "project"): string { + return `skill:${source}:${skillName}` + } + + private buildSkillContent(skillName: string, description: string, toolNames: string[]): string { + const title = skillName + .split("-") + .map((segment) => segment.charAt(0).toUpperCase() + segment.slice(1)) + .join(" ") + const bulletList = toolNames.map((toolName) => "- `" + toolName + "`").join("\n") + const inlineTools = toolNames.map((toolName) => "`" + toolName + "`").join(" then ") + + return `--- +name: ${skillName} +description: ${description} +--- + +# ${title} + +## When to use + +${description} + +## Preferred tools + +${bulletList} + +## Workflow + +1. Start with ${inlineTools}. +2. Keep the sequence focused on the same reusable workflow. +3. Update this skill when the workflow changes materially. +` + } +} diff --git a/src/services/self-improving/LearningStore.ts b/src/services/self-improving/LearningStore.ts new file mode 100644 index 0000000000..c9b1c04c50 --- /dev/null +++ b/src/services/self-improving/LearningStore.ts @@ -0,0 +1,389 @@ +import * as fs from "fs/promises" +import * as path from "path" +import crypto from "crypto" + +import { safeWriteJson } from "../../utils/safeWriteJson" +import type { ImprovementAction, LearnedPattern, LearningConfig, LearningEvent, LearningState, Logger } from "./types" +import { EMPTY_STATE } from "./types" + +/** + * File names for the learning store + */ +const STATE_FILE = "state.json" +const PATTERNS_DIR = "patterns" +const ARCHIVE_DIR = "archive" +const PATTERN_INDEX_FILE = "_index.json" + +/** + * LearningStore - atomic file-based persistence for learning state. + * + * Storage layout: + * globalStorage/self-improving/ + * state.json - canonical LearningState metadata + counters + * patterns/ - per-pattern source-of-truth files + * _index.json - compact index of active/stale/archived patterns + * .json - individual pattern data + * archive/ - cold storage for archived patterns + * .json + */ +export class LearningStore { + private readonly baseDir: string + private readonly patternsDir: string + private readonly archiveDir: string + private state: LearningState + private readonly logger: Logger + private initialized = false + + constructor(baseDir: string, logger: Logger) { + this.baseDir = path.join(baseDir, "self-improving") + this.patternsDir = path.join(this.baseDir, PATTERNS_DIR) + this.archiveDir = path.join(this.baseDir, ARCHIVE_DIR) + this.state = this.createEmptyState() + this.logger = logger + } + + /** + * Initialize the store - create directories and load persisted state. + * If state is corrupted or missing, falls back to empty defaults. + */ + async initialize(): Promise { + if (this.initialized) { + return + } + + try { + await fs.mkdir(this.patternsDir, { recursive: true }) + await fs.mkdir(this.archiveDir, { recursive: true }) + await this.loadState() + this.initialized = true + } catch (error) { + this.logger.appendLine( + `[LearningStore] Initialization error: ${error instanceof Error ? error.message : String(error)}`, + ) + this.state = this.createEmptyState() + this.initialized = true + } + } + + /** + * Load state from disk with graceful degradation. + */ + private async loadState(): Promise { + const statePath = path.join(this.baseDir, STATE_FILE) + + try { + const raw = await fs.readFile(statePath, "utf-8") + const parsed = JSON.parse(raw) as Partial + + if (parsed && typeof parsed === "object" && parsed.version === 1) { + this.state = this.mergeWithDefaults(parsed) + await this.loadPatternFiles() + this.logger.appendLine( + `[LearningStore] Loaded state: ${this.state.patterns.length} patterns, ${this.state.recentEvents.length} events`, + ) + return + } + + this.logger.appendLine("[LearningStore] Invalid state version, using defaults") + } catch (error: unknown) { + const errorCode = typeof error === "object" && error !== null && "code" in error ? error.code : undefined + if (errorCode === "ENOENT") { + this.logger.appendLine("[LearningStore] No existing state, starting fresh") + } else { + this.logger.appendLine( + `[LearningStore] Corrupted state (${error instanceof Error ? error.message : String(error)}), using defaults`, + ) + } + } + + this.state = this.createEmptyState() + } + + private async loadPatternFiles(): Promise { + const activePatterns = await this.readPatternDirectory(this.patternsDir) + const archivedPatterns = await this.readPatternDirectory(this.archiveDir) + + if (activePatterns.length > 0) { + this.state.patterns = activePatterns + } + + if (archivedPatterns.length > 0) { + this.state.archivedPatterns = archivedPatterns.map((pattern) => ({ + ...pattern, + state: "archived", + })) + } + } + + private async readPatternDirectory(directoryPath: string): Promise { + try { + const entries = await fs.readdir(directoryPath, { withFileTypes: true }) + const patterns: LearnedPattern[] = [] + + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith(".json") || entry.name === PATTERN_INDEX_FILE) { + continue + } + + try { + const raw = await fs.readFile(path.join(directoryPath, entry.name), "utf-8") + const parsed = JSON.parse(raw) as LearnedPattern + if (parsed?.id) { + patterns.push(parsed) + } + } catch (error) { + this.logger.appendLine( + `[LearningStore] Failed to read pattern ${entry.name}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + + return patterns + } catch { + return [] + } + } + + /** + * Merge parsed state with defaults to handle schema evolution. + */ + private mergeWithDefaults(parsed: Partial): LearningState { + const config = { ...EMPTY_STATE.config, ...(parsed.config ?? {}) } + + return { + version: 1, + config, + counters: { + userTurnsSinceReview: parsed.counters?.userTurnsSinceReview ?? 0, + toolIterationsSinceReview: parsed.counters?.toolIterationsSinceReview ?? 0, + }, + patterns: Array.isArray(parsed.patterns) ? parsed.patterns : [], + archivedPatterns: Array.isArray(parsed.archivedPatterns) ? parsed.archivedPatterns : [], + recentEvents: Array.isArray(parsed.recentEvents) ? parsed.recentEvents.slice(-config.maxStoredEvents) : [], + pendingActions: Array.isArray(parsed.pendingActions) ? parsed.pendingActions : [], + telemetry: { + promptEnrichmentUses: parsed.telemetry?.promptEnrichmentUses ?? 0, + toolPreferenceUses: parsed.telemetry?.toolPreferenceUses ?? 0, + errorAvoidanceUses: parsed.telemetry?.errorAvoidanceUses ?? 0, + skillSuggestionCount: parsed.telemetry?.skillSuggestionCount ?? 0, + lastReviewAt: parsed.telemetry?.lastReviewAt, + lastCuratorRunAt: parsed.telemetry?.lastCuratorRunAt, + }, + } + } + + /** + * Persist the full state to disk with state.json committed last. + */ + async persist(): Promise { + if (!this.initialized) { + return + } + + try { + this.enforceBounds() + + await this.persistPatternFiles(this.patternsDir, this.state.patterns) + await this.persistPatternFiles(this.archiveDir, this.state.archivedPatterns) + await this.writePatternIndex() + await safeWriteJson(path.join(this.baseDir, STATE_FILE), this.state, { prettyPrint: true }) + } catch (error) { + this.logger.appendLine( + `[LearningStore] Persist error: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + + private async persistPatternFiles(directoryPath: string, patterns: readonly LearnedPattern[]): Promise { + const expectedNames = new Set(patterns.map((pattern) => `${pattern.id}.json`)) + + await Promise.all( + patterns.map((pattern) => + safeWriteJson(path.join(directoryPath, `${pattern.id}.json`), pattern, { prettyPrint: true }), + ), + ) + + try { + const existingEntries = await fs.readdir(directoryPath, { withFileTypes: true }) + await Promise.all( + existingEntries + .filter( + (entry) => + entry.isFile() && + entry.name.endsWith(".json") && + entry.name !== PATTERN_INDEX_FILE && + !expectedNames.has(entry.name), + ) + .map((entry) => fs.rm(path.join(directoryPath, entry.name), { force: true })), + ) + } catch (error) { + this.logger.appendLine( + `[LearningStore] Pattern cleanup error: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + + private async writePatternIndex(): Promise { + await safeWriteJson( + path.join(this.patternsDir, PATTERN_INDEX_FILE), + { + version: 1, + updatedAt: Date.now(), + activePatternIds: this.state.patterns.map((pattern) => pattern.id), + archivedPatternIds: this.state.archivedPatterns.map((pattern) => pattern.id), + }, + { prettyPrint: true }, + ) + } + + /** + * Enforce storage bounds (max patterns, max events). + */ + private enforceBounds(): void { + const maxPatterns = this.state.config.maxStoredPatterns + const maxEvents = this.state.config.maxStoredEvents + + if (this.state.patterns.length > maxPatterns) { + this.state.patterns = [...this.state.patterns] + .sort((a, b) => a.confidenceScore - b.confidenceScore) + .slice(-maxPatterns) + } + + if (this.state.archivedPatterns.length > maxPatterns) { + this.state.archivedPatterns = [...this.state.archivedPatterns] + .sort((a, b) => a.lastSeenAt - b.lastSeenAt) + .slice(-maxPatterns) + } + + if (this.state.recentEvents.length > maxEvents) { + this.state.recentEvents = this.state.recentEvents.slice(-maxEvents) + } + } + + // ──── Getters ──── + + getState(): Readonly { + return this.state + } + + getConfig(): Readonly { + return this.state.config + } + + getPatterns(): readonly LearnedPattern[] { + return this.state.patterns + } + + getArchivedPatterns(): readonly LearnedPattern[] { + return this.state.archivedPatterns + } + + getRecentEvents(): readonly LearningEvent[] { + return this.state.recentEvents + } + + getPendingActions(): readonly ImprovementAction[] { + return this.state.pendingActions + } + + getTelemetry(): Readonly { + return this.state.telemetry + } + + getCounters(): Readonly { + return this.state.counters + } + + // ──── Mutations ──── + + setConfig(config: Partial): void { + this.state.config = { ...this.state.config, ...config } + } + + addEvent(event: LearningEvent): void { + this.state.recentEvents.push(event) + } + + addPattern(pattern: LearnedPattern): void { + const existing = this.state.patterns.findIndex((candidate) => candidate.id === pattern.id) + if (existing >= 0) { + this.state.patterns[existing] = pattern + return + } + + this.state.patterns.push(pattern) + } + + updatePattern(id: string, updates: Partial): void { + const index = this.state.patterns.findIndex((pattern) => pattern.id === id) + if (index >= 0) { + this.state.patterns[index] = { ...this.state.patterns[index], ...updates } + } + } + + removePattern(id: string): void { + this.state.patterns = this.state.patterns.filter((pattern) => pattern.id !== id) + } + + archivePattern(id: string): void { + const index = this.state.patterns.findIndex((pattern) => pattern.id === id) + if (index >= 0) { + const pattern = { ...this.state.patterns[index], state: "archived" as const } + this.state.archivedPatterns.push(pattern) + this.state.patterns.splice(index, 1) + } + } + + addAction(action: ImprovementAction): void { + this.state.pendingActions.push(action) + } + + removeAction(id: string): void { + this.state.pendingActions = this.state.pendingActions.filter((action) => action.id !== id) + } + + incrementUserTurns(): void { + this.state.counters.userTurnsSinceReview++ + } + + incrementToolIterations(delta = 1): void { + this.state.counters.toolIterationsSinceReview += delta + } + + resetCounters(): void { + this.state.counters.userTurnsSinceReview = 0 + this.state.counters.toolIterationsSinceReview = 0 + } + + updateTelemetry(updates: Partial): void { + this.state.telemetry = { ...this.state.telemetry, ...updates } + } + + /** + * Reset all learning state to defaults. + */ + async reset(): Promise { + this.state = this.createEmptyState() + await this.persist() + } + + private createEmptyState(): LearningState { + return { + ...EMPTY_STATE, + config: { ...EMPTY_STATE.config }, + counters: { ...EMPTY_STATE.counters }, + patterns: [], + archivedPatterns: [], + recentEvents: [], + pendingActions: [], + telemetry: { ...EMPTY_STATE.telemetry }, + } + } + + /** + * Generate a unique ID for patterns, events, and actions. + */ + static generateId(): string { + return crypto.randomUUID() + } +} diff --git a/src/services/self-improving/MemoryBackend.ts b/src/services/self-improving/MemoryBackend.ts new file mode 100644 index 0000000000..44a47e67ed --- /dev/null +++ b/src/services/self-improving/MemoryBackend.ts @@ -0,0 +1,42 @@ +import type { MemoryEntry } from "@roo-code/types" + +/** + * MemoryBackend — abstract interface for memory storage backends. + * + * Both the built-in MemoryStore and the optional agentmemory adapter + * implement this interface, allowing the SelfImprovingManager to + * switch between backends transparently. + */ +export interface MemoryBackend { + /** Initialize the backend */ + initialize(): Promise + + /** Store a memory entry */ + store(entry: Omit): Promise + + /** Search memory entries by query */ + search(query: string, maxResults?: number): Promise + + /** Recall recent memory entries */ + recall(maxResults?: number): Promise + + /** Remove a memory entry by ID */ + forget(id: string): Promise + + /** Remove entries matching a substring */ + forgetByContent(substring: string): Promise + + /** Get backend statistics */ + getStats(): Promise<{ entryCount: number; backend: string }> + + /** Clear all entries */ + clear(): Promise + + /** Dispose the backend */ + dispose(): Promise +} + +/** + * MemoryBackendType — supported backend implementations + */ +export type MemoryBackendType = "builtin" | "agentmemory" diff --git a/src/services/self-improving/MemoryBackendFactory.ts b/src/services/self-improving/MemoryBackendFactory.ts new file mode 100644 index 0000000000..b2e04923ba --- /dev/null +++ b/src/services/self-improving/MemoryBackendFactory.ts @@ -0,0 +1,32 @@ +import { AgentMemoryAdapter } from "./AgentMemoryAdapter" +import type { MemoryBackend, MemoryBackendType } from "./MemoryBackend" +import { MemoryStore } from "./MemoryStore" +import type { Logger } from "./types" + +/** + * MemoryBackendFactory — creates the appropriate memory backend + * based on configuration. + * + * Supports: + * - "builtin" (default): Zoo-Code's own MemoryStore + * - "agentmemory": agentmemory REST API adapter + */ +export class MemoryBackendFactory { + /** + * Create a memory backend. + * + * @param type - Backend type ("builtin" | "agentmemory") + * @param baseDir - Base directory for built-in storage + * @param logger - Logger instance + * @param agentMemoryUrl - Optional agentmemory server URL + */ + static create(type: MemoryBackendType, baseDir: string, logger: Logger, agentMemoryUrl?: string): MemoryBackend { + switch (type) { + case "agentmemory": + return new AgentMemoryAdapter(logger, agentMemoryUrl) + case "builtin": + default: + return new MemoryStore(baseDir, logger) + } + } +} diff --git a/src/services/self-improving/MemoryStore.ts b/src/services/self-improving/MemoryStore.ts new file mode 100644 index 0000000000..58f62b2e92 --- /dev/null +++ b/src/services/self-improving/MemoryStore.ts @@ -0,0 +1,605 @@ +import * as fs from "fs/promises" +import * as path from "path" +import crypto from "crypto" + +import { safeWriteJson } from "../../utils/safeWriteJson" +import type { MemoryContext, MemoryEntry } from "@roo-code/types" +import type { MemoryBackend, MemoryBackendType } from "./MemoryBackend" +import type { Logger } from "./types" + +/** + * Store type for memory categorization. + * Mirrors Hermes' MEMORY.md (environment) vs USER.md (user profile) split. + */ +export type MemoryStoreType = "environment" | "userProfile" + +const MEMORY_SOURCES: ReadonlySet = new Set(["learning", "user", "system", "review"]) + +/** + * MemoryStore - real bounded memory subsystem. + * + * Implements Hermes' dual-store approach: + * - environment: durable operational facts, project knowledge, learned patterns + * - userProfile: user preferences, corrections, style feedback + * + * Key design: + * - Frozen snapshot at session start (prompt stability) + * - Live writes go to disk but NOT to active snapshot + * - Exact-duplicate rejection on load + * - Substring-based replace and remove + * - Bounded retention per store + */ +export class MemoryStore implements MemoryBackend { + private readonly baseDir: string + private readonly logger: Logger + private environment: MemoryEntry[] = [] + private userProfile: MemoryEntry[] = [] + private environmentSnapshot: MemoryEntry[] = [] + private userProfileSnapshot: MemoryEntry[] = [] + private revision = 0 + private initialized = false + + private static readonly MAX_ENVIRONMENT_ENTRIES = 50 + private static readonly MAX_USER_PROFILE_ENTRIES = 20 + private static readonly MAX_ENTRY_LENGTH = 2000 + private static readonly MAX_SNAPSHOT_ENVIRONMENT_ENTRIES = 5 + private static readonly MAX_SNAPSHOT_USER_PROFILE_ENTRIES = 5 + + constructor(baseDir: string, logger: Logger) { + this.baseDir = path.join(baseDir, "self-improving", "memory") + this.logger = logger + } + + get backendType(): MemoryBackendType { + return "builtin" + } + + /** + * Initialize the memory store - load persisted entries from disk. + */ + async initialize(): Promise { + if (this.initialized) { + return + } + + try { + await fs.mkdir(this.baseDir, { recursive: true }) + await this.loadFromDisk() + this.logger.appendLine( + `[MemoryStore] Initialized: ${this.environment.length} environment, ${this.userProfile.length} user profile entries`, + ) + } catch (error) { + this.logger.appendLine( + `[MemoryStore] Initialization error: ${error instanceof Error ? error.message : String(error)}`, + ) + } finally { + this.initialized = true + } + } + + async store(entry: Omit): Promise { + return this.addEnvironmentEntry(entry.content, { + source: entry.source, + tags: entry.tags, + expiresAt: entry.expiresAt, + }) + } + + async search(query: string, maxResults: number = 10): Promise { + await this.ensureInitialized() + + const lowerQuery = query.toLowerCase() + const allEntries = [...this.environment, ...this.userProfile] + return allEntries + .filter((entry) => entry.content.toLowerCase().includes(lowerQuery)) + .slice(0, maxResults) + .map((entry) => this.cloneEntry(entry)) + } + + async recall(maxResults: number = 20): Promise { + await this.ensureInitialized() + + return [...this.environment, ...this.userProfile] + .sort((left, right) => { + const leftTimestamp = left.updatedAt ?? left.createdAt + const rightTimestamp = right.updatedAt ?? right.createdAt + return rightTimestamp - leftTimestamp + }) + .slice(0, maxResults) + .map((entry) => this.cloneEntry(entry)) + } + + async forget(id: string): Promise { + await this.ensureInitialized() + + const envIdx = this.environment.findIndex((entry) => entry.id === id) + if (envIdx >= 0) { + this.environment.splice(envIdx, 1) + await this.persistStore("environment") + return true + } + + const userIdx = this.userProfile.findIndex((entry) => entry.id === id) + if (userIdx >= 0) { + this.userProfile.splice(userIdx, 1) + await this.persistStore("userProfile") + return true + } + + return false + } + + async forgetByContent(substring: string): Promise { + await this.ensureInitialized() + + const lowerSubstring = substring.trim().toLowerCase() + if (!lowerSubstring) { + return 0 + } + + let removed = 0 + + const envBefore = this.environment.length + this.environment = this.environment.filter((entry) => !entry.content.toLowerCase().includes(lowerSubstring)) + removed += envBefore - this.environment.length + + const userBefore = this.userProfile.length + this.userProfile = this.userProfile.filter((entry) => !entry.content.toLowerCase().includes(lowerSubstring)) + removed += userBefore - this.userProfile.length + + if (removed > 0) { + await this.persistStore("environment") + await this.persistStore("userProfile") + } + + return removed + } + + /** + * Load entries from disk with duplicate rejection. + */ + private async loadFromDisk(): Promise { + this.environment = await this.loadStoreFile("environment") + this.userProfile = await this.loadStoreFile("userProfile") + this.takeSnapshot() + } + + /** + * Load a single store file with validation and dedup. + */ + private async loadStoreFile(type: MemoryStoreType): Promise { + try { + const raw = await fs.readFile(this.getFilePath(type), "utf-8") + const parsed = JSON.parse(raw) + + if (!Array.isArray(parsed)) { + return [] + } + + const seen = new Set() + const deduped: MemoryEntry[] = [] + + for (const candidate of parsed) { + const entry = this.sanitizePersistedEntry(candidate) + if (!entry) { + continue + } + + const contentKey = this.normalizeContent(entry.content) + if (seen.has(contentKey)) { + continue + } + + seen.add(contentKey) + deduped.push(entry) + } + + return this.enforceBounds(type, deduped) + } catch (error: unknown) { + const errorCode = typeof error === "object" && error !== null && "code" in error ? error.code : undefined + if (errorCode !== "ENOENT") { + this.logger.appendLine( + `[MemoryStore] Load error for ${this.getFilePath(type)}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + + return [] + } + } + + /** + * Take a frozen snapshot of current memory for prompt injection. + * Live writes update the working store but NOT the snapshot. + */ + takeSnapshot(): void { + this.environmentSnapshot = this.environment.map((entry) => this.cloneEntry(entry)) + this.userProfileSnapshot = this.userProfile.map((entry) => this.cloneEntry(entry)) + this.revision += 1 + } + + /** + * Get the frozen snapshot context for prompt injection. + */ + getSnapshotContext(): MemoryContext { + return { + entries: this.buildSnapshotEntries(), + revision: this.revision, + generatedAt: Date.now(), + } + } + + /** + * Get snapshot as formatted string for prompt injection. + */ + getSnapshotString(): string { + const context = this.getSnapshotContext() + if (context.entries.length === 0) return "" + + const lines = context.entries.map((entry) => { + // Sanitize: single line, strip control characters, no markdown headings + const sanitized = entry.content + .split("") + .filter((c) => { + const code = c.charCodeAt(0) + return code >= 32 || code === 9 || code === 10 || code === 13 + }) + .join("") + .replace(/\n/g, " ") + .replace(/^#+\s*/gm, "") + .trim() + const tags = entry.tags?.length ? ` [${entry.tags.join(", ")}]` : "" + return `- ${sanitized}${tags}` + }) + + return `\n## Learned Context\n${lines.join("\n")}\n` + } + + // ──── Environment store operations ──── + + /** + * Add an entry to the environment store. + * Rejects exact duplicates. Persists to disk but does NOT update the snapshot. + */ + async addEnvironmentEntry( + content: string, + options?: { + source?: MemoryEntry["source"] + tags?: string[] + expiresAt?: number + }, + ): Promise { + return this.addEntry("environment", content, options) + } + + /** + * Replace entries in the environment store that contain a substring. + * If no match is found, adds as new entry. + */ + async replaceEnvironmentEntry( + substring: string, + newContent: string, + options?: { + source?: MemoryEntry["source"] + tags?: string[] + }, + ): Promise { + return this.replaceEntry("environment", substring, newContent, options) + } + + /** + * Remove entries from the environment store that contain a substring. + */ + async removeEnvironmentEntry(substring: string): Promise { + return this.removeEntry("environment", substring) + } + + // ──── User profile store operations ──── + + /** + * Add an entry to the user profile store. + */ + async addUserProfileEntry( + content: string, + options?: { + source?: MemoryEntry["source"] + tags?: string[] + expiresAt?: number + }, + ): Promise { + return this.addEntry("userProfile", content, options) + } + + /** + * Replace entries in the user profile store that contain a substring. + */ + async replaceUserProfileEntry( + substring: string, + newContent: string, + options?: { + source?: MemoryEntry["source"] + tags?: string[] + }, + ): Promise { + return this.replaceEntry("userProfile", substring, newContent, options) + } + + /** + * Remove entries from the user profile store that contain a substring. + */ + async removeUserProfileEntry(substring: string): Promise { + return this.removeEntry("userProfile", substring) + } + + // ──── Generic store operations ──── + + private getStore(type: MemoryStoreType): MemoryEntry[] { + return type === "environment" ? this.environment : this.userProfile + } + + private setStore(type: MemoryStoreType, entries: MemoryEntry[]): void { + if (type === "environment") { + this.environment = entries + return + } + + this.userProfile = entries + } + + private getMaxEntries(type: MemoryStoreType): number { + return type === "environment" ? MemoryStore.MAX_ENVIRONMENT_ENTRIES : MemoryStore.MAX_USER_PROFILE_ENTRIES + } + + private getFilePath(type: MemoryStoreType): string { + return path.join(this.baseDir, type === "environment" ? "environment.json" : "user-profile.json") + } + + private async ensureInitialized(): Promise { + if (!this.initialized) { + await this.initialize() + } + } + + private async addEntry( + type: MemoryStoreType, + content: string, + options?: { + source?: MemoryEntry["source"] + tags?: string[] + expiresAt?: number + }, + ): Promise { + await this.ensureInitialized() + + const trimmed = content.trim() + if (!trimmed || trimmed.length > MemoryStore.MAX_ENTRY_LENGTH) { + return null + } + + const normalized = this.normalizeContent(trimmed) + const store = this.getStore(type) + if (store.some((entry) => this.normalizeContent(entry.content) === normalized)) { + return null + } + + const now = Date.now() + const entry: MemoryEntry = { + id: crypto.randomUUID(), + content: trimmed, + source: this.normalizeSource(options?.source), + createdAt: now, + updatedAt: now, + relevanceScore: 1, + tags: this.normalizeTags(options?.tags), + expiresAt: typeof options?.expiresAt === "number" ? options.expiresAt : undefined, + } + + this.setStore(type, this.enforceBounds(type, [...store, entry])) + await this.persistStore(type) + + return this.cloneEntry(entry) + } + + private async replaceEntry( + type: MemoryStoreType, + substring: string, + newContent: string, + options?: { + source?: MemoryEntry["source"] + tags?: string[] + }, + ): Promise { + await this.ensureInitialized() + + const trimmedContent = newContent.trim() + if (!trimmedContent || trimmedContent.length > MemoryStore.MAX_ENTRY_LENGTH) { + throw new Error("Replacement memory content must be non-empty and within bounds") + } + + const normalizedSubstring = substring.trim().toLowerCase() + const store = this.getStore(type) + const remaining = + normalizedSubstring.length > 0 + ? store.filter((entry) => !entry.content.toLowerCase().includes(normalizedSubstring)) + : [...store] + + const duplicate = remaining.find( + (entry) => this.normalizeContent(entry.content) === this.normalizeContent(trimmedContent), + ) + + if (duplicate) { + this.setStore(type, this.enforceBounds(type, remaining)) + if (remaining.length !== store.length) { + await this.persistStore(type) + } + + return this.cloneEntry(duplicate) + } + + const now = Date.now() + const entry: MemoryEntry = { + id: crypto.randomUUID(), + content: trimmedContent, + source: this.normalizeSource(options?.source), + createdAt: now, + updatedAt: now, + relevanceScore: 1, + tags: this.normalizeTags(options?.tags), + } + + this.setStore(type, this.enforceBounds(type, [...remaining, entry])) + await this.persistStore(type) + + return this.cloneEntry(entry) + } + + private async removeEntry(type: MemoryStoreType, substring: string): Promise { + await this.ensureInitialized() + + const normalizedSubstring = substring.trim().toLowerCase() + if (!normalizedSubstring) { + return false + } + + const store = this.getStore(type) + const remaining = store.filter((entry) => !entry.content.toLowerCase().includes(normalizedSubstring)) + const removed = remaining.length !== store.length + + if (!removed) { + return false + } + + this.setStore(type, remaining) + await this.persistStore(type) + + return true + } + + private async persistStore(type: MemoryStoreType): Promise { + try { + await safeWriteJson(this.getFilePath(type), this.getStore(type), { prettyPrint: true }) + } catch (error) { + this.logger.appendLine( + `[MemoryStore] Persist error for ${type}: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + + private sanitizePersistedEntry(value: unknown): MemoryEntry | null { + if (!value || typeof value !== "object") { + return null + } + + const candidate = value as Partial + const trimmedContent = typeof candidate.content === "string" ? candidate.content.trim() : "" + if (!trimmedContent) { + return null + } + + const now = Date.now() + const createdAt = typeof candidate.createdAt === "number" ? candidate.createdAt : now + const updatedAt = typeof candidate.updatedAt === "number" ? candidate.updatedAt : createdAt + + return { + id: typeof candidate.id === "string" && candidate.id.trim().length > 0 ? candidate.id : crypto.randomUUID(), + content: trimmedContent.slice(0, MemoryStore.MAX_ENTRY_LENGTH), + source: this.normalizeSource(candidate.source), + createdAt, + updatedAt, + relevanceScore: + typeof candidate.relevanceScore === "number" + ? Math.min(1, Math.max(0, candidate.relevanceScore)) + : undefined, + tags: this.normalizeTags(candidate.tags), + expiresAt: typeof candidate.expiresAt === "number" ? candidate.expiresAt : undefined, + } + } + + private buildSnapshotEntries(): MemoryEntry[] { + const environmentEntries = this.environmentSnapshot + .slice(-MemoryStore.MAX_SNAPSHOT_ENVIRONMENT_ENTRIES) + .map((entry) => this.cloneEntry(entry)) + const userProfileEntries = this.userProfileSnapshot + .slice(-MemoryStore.MAX_SNAPSHOT_USER_PROFILE_ENTRIES) + .map((entry) => this.cloneEntry(entry)) + + return [...environmentEntries, ...userProfileEntries] + } + + private enforceBounds(type: MemoryStoreType, entries: MemoryEntry[]): MemoryEntry[] { + const maxEntries = this.getMaxEntries(type) + if (entries.length <= maxEntries) { + return entries + } + + return [...entries].sort((left, right) => left.createdAt - right.createdAt).slice(-maxEntries) + } + + private normalizeContent(content: string): string { + return content.trim().toLowerCase() + } + + private normalizeSource(source: MemoryEntry["source"] | undefined): MemoryEntry["source"] { + return source && MEMORY_SOURCES.has(source) ? source : "learning" + } + + private normalizeTags(tags: string[] | undefined): string[] | undefined { + if (!Array.isArray(tags)) { + return undefined + } + + const normalized = Array.from(new Set(tags.map((tag) => tag.trim()).filter((tag) => tag.length > 0))) + + return normalized.length > 0 ? normalized : undefined + } + + private cloneEntry(entry: MemoryEntry): MemoryEntry { + return { + ...entry, + tags: entry.tags ? [...entry.tags] : undefined, + } + } + + /** + * Get count of entries per store. + */ + async getStats(): Promise<{ entryCount: number; backend: string }> { + await this.ensureInitialized() + + return { + entryCount: this.environment.length + this.userProfile.length, + backend: "builtin", + } + } + + async clear(): Promise { + await this.reset() + } + + /** + * Reset all memory stores. + */ + async reset(): Promise { + await this.ensureInitialized() + + this.environment = [] + this.userProfile = [] + this.environmentSnapshot = [] + this.userProfileSnapshot = [] + this.revision = 0 + + try { + await Promise.all([ + safeWriteJson(this.getFilePath("environment"), [], { prettyPrint: true }), + safeWriteJson(this.getFilePath("userProfile"), [], { prettyPrint: true }), + ]) + } catch (error) { + this.logger.appendLine( + `[MemoryStore] Reset error: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + + async dispose(): Promise { + this.initialized = false + } +} diff --git a/src/services/self-improving/PatternAnalyzer.ts b/src/services/self-improving/PatternAnalyzer.ts new file mode 100644 index 0000000000..c545187c28 --- /dev/null +++ b/src/services/self-improving/PatternAnalyzer.ts @@ -0,0 +1,303 @@ +import crypto from "crypto" + +import type { LearnedPattern, LearningEvent } from "./types" + +/** + * PatternAnalyzer - extracts learned patterns from event streams + * using deterministic heuristics (frequency analysis, correction tracking, + * success correlation). + * + * Adapted from Hermes' symbolic pattern extraction approach. + */ +export class PatternAnalyzer { + /** + * Analyze a batch of events and return new/updated patterns. + * This is the main entry point called during review cycles. + */ + analyze(events: LearningEvent[], existingPatterns: LearnedPattern[]): LearnedPattern[] { + const patterns: LearnedPattern[] = [] + const now = Date.now() + + const correctionPatterns = this.extractCorrectionPatterns(events, existingPatterns, now) + patterns.push(...correctionPatterns) + + const successPatterns = this.extractSuccessPatterns(events, existingPatterns, now) + patterns.push(...successPatterns) + + const toolPatterns = this.extractToolPatterns(events, existingPatterns, now) + patterns.push(...toolPatterns) + + const codeIndexPatterns = this.extractCodeIndexPatterns(events, existingPatterns, now) + patterns.push(...codeIndexPatterns) + + return patterns + } + + /** + * Extract error-avoidance patterns from correction/failure events. + */ + private extractCorrectionPatterns( + events: LearningEvent[], + existingPatterns: LearnedPattern[], + now: number, + ): LearnedPattern[] { + const patterns: LearnedPattern[] = [] + const correctionEvents = events.filter( + (event) => + event.signal === "USER_CORRECTION" || (event.signal === "TASK_FAILURE" && event.outcome.corrected), + ) + + const byErrorKey = new Map() + for (const event of correctionEvents) { + const key = event.context.errorKey || "unknown" + const bucket = byErrorKey.get(key) ?? [] + bucket.push(event) + byErrorKey.set(key, bucket) + } + + for (const [errorKey, errorEvents] of byErrorKey) { + const frequency = errorEvents.length + const existing = existingPatterns.find( + (pattern) => pattern.patternType === "error" && pattern.context.errorKeys?.includes(errorKey), + ) + + if (existing) { + patterns.push({ + ...existing, + frequency: existing.frequency + frequency, + lastSeenAt: now, + confidenceScore: Math.min(1, existing.confidenceScore + frequency * 0.05), + successRate: Math.max(0, existing.successRate - frequency * 0.02), + }) + } else if (frequency >= 2) { + patterns.push({ + id: crypto.randomUUID(), + patternType: "error", + state: "active", + summary: `Avoid: repeated ${errorKey} errors detected`, + confidenceScore: Math.min(0.5, frequency * 0.1), + frequency, + successRate: 0.3, + firstSeenAt: now, + lastSeenAt: now, + sourceSignals: ["USER_CORRECTION", "TASK_FAILURE"], + context: { + errorKeys: [errorKey], + toolNames: this.collectToolNames(errorEvents), + }, + }) + } + } + + return patterns + } + + /** + * Extract success patterns from task success events. + */ + private extractSuccessPatterns( + events: LearningEvent[], + existingPatterns: LearnedPattern[], + now: number, + ): LearnedPattern[] { + const patterns: LearnedPattern[] = [] + const successEvents = events.filter((event) => event.signal === "TASK_SUCCESS") + + if (successEvents.length < 3) { + return patterns + } + + const byToolSet = new Map() + for (const event of successEvents) { + const toolKey = [...(event.context.toolNames ?? [])].sort().join(",") + if (!toolKey) { + continue + } + + const bucket = byToolSet.get(toolKey) ?? [] + bucket.push(event) + byToolSet.set(toolKey, bucket) + } + + for (const [toolKey, toolEvents] of byToolSet) { + const frequency = toolEvents.length + const existing = existingPatterns.find( + (pattern) => pattern.patternType === "tool" && this.hasMatchingToolNames(pattern, toolKey.split(",")), + ) + + if (existing) { + patterns.push({ + ...existing, + frequency: existing.frequency + frequency, + lastSeenAt: now, + confidenceScore: Math.min(1, existing.confidenceScore + frequency * 0.03), + successRate: Math.min(1, existing.successRate + frequency * 0.02), + }) + } else if (frequency >= 3) { + patterns.push({ + id: crypto.randomUUID(), + patternType: "tool", + state: "active", + summary: `Effective tool combination: ${toolKey}`, + confidenceScore: Math.min(0.6, frequency * 0.1), + frequency, + successRate: 0.7, + firstSeenAt: now, + lastSeenAt: now, + sourceSignals: ["TASK_SUCCESS"], + context: { + toolNames: toolKey.split(","), + }, + }) + } + } + + return patterns + } + + /** + * Extract tool preference patterns. + */ + private extractToolPatterns( + events: LearningEvent[], + existingPatterns: LearnedPattern[], + now: number, + ): LearnedPattern[] { + const patterns: LearnedPattern[] = [] + const toolCounts = new Map() + + for (const event of events) { + for (const toolName of event.context.toolNames ?? []) { + const counts = toolCounts.get(toolName) ?? { success: 0, failure: 0 } + if (event.signal === "TASK_SUCCESS") { + counts.success++ + } else if (event.signal === "TASK_FAILURE") { + counts.failure++ + } + toolCounts.set(toolName, counts) + } + } + + for (const [toolName, counts] of toolCounts) { + const total = counts.success + counts.failure + if (total < 3) { + continue + } + + const successRate = counts.success / total + const existing = existingPatterns.find( + (pattern) => pattern.patternType === "prompt" && this.hasMatchingToolNames(pattern, [toolName]), + ) + + if (existing) { + const combinedFrequency = existing.frequency + total + const existingSuccesses = existing.successRate * existing.frequency + const combinedSuccessRate = (existingSuccesses + counts.success) / combinedFrequency + + patterns.push({ + ...existing, + frequency: combinedFrequency, + lastSeenAt: now, + successRate: combinedSuccessRate, + confidenceScore: Math.min(1, existing.confidenceScore + 0.02), + }) + } else if (successRate > 0.7) { + patterns.push({ + id: crypto.randomUUID(), + patternType: "prompt", + state: "active", + summary: `Prefer ${toolName} for reliable results`, + confidenceScore: Math.min(0.5, successRate * 0.5), + frequency: total, + successRate, + firstSeenAt: now, + lastSeenAt: now, + sourceSignals: ["TASK_SUCCESS"], + context: { + toolNames: [toolName], + }, + }) + } + } + + return patterns + } + + /** + * Extract code index correlation patterns. + */ + private extractCodeIndexPatterns( + events: LearningEvent[], + existingPatterns: LearnedPattern[], + now: number, + ): LearnedPattern[] { + const codeIndexEvents = events.filter((event) => event.signal === "CODE_INDEX_HIT") + if (codeIndexEvents.length < 3) { + return [] + } + + const totalHits = codeIndexEvents.reduce((sum, event) => sum + (event.context.codeIndex?.hits ?? 0), 0) + const averageHits = totalHits / codeIndexEvents.length + if (averageHits <= 0) { + return [] + } + + const summary = `Code indexing correlates with task outcomes (avg ${averageHits.toFixed(1)} hits/event)` + const existing = existingPatterns.find( + (pattern) => pattern.patternType === "code-index" && pattern.summary === summary, + ) + + if (existing) { + return [ + { + ...existing, + frequency: existing.frequency + codeIndexEvents.length, + lastSeenAt: now, + confidenceScore: Math.min(1, existing.confidenceScore + averageHits * 0.01), + }, + ] + } + + return [ + { + id: crypto.randomUUID(), + patternType: "code-index", + state: "active", + summary, + confidenceScore: Math.min(0.5, averageHits * 0.05), + frequency: codeIndexEvents.length, + successRate: 0.6, + firstSeenAt: now, + lastSeenAt: now, + sourceSignals: ["CODE_INDEX_HIT"], + context: {}, + }, + ] + } + + /** + * Collect unique tool names from a set of events. + */ + private collectToolNames(events: LearningEvent[]): string[] { + const names = new Set() + for (const event of events) { + for (const name of event.context.toolNames ?? []) { + names.add(name) + } + } + + return [...names] + } + + private hasMatchingToolNames(pattern: LearnedPattern, toolNames: string[]): boolean { + const existingToolNames = pattern.context.toolNames + if (!existingToolNames || existingToolNames.length !== toolNames.length) { + return false + } + + const normalizedExisting = [...existingToolNames].sort() + const normalizedIncoming = [...toolNames].sort() + + return normalizedExisting.every((toolName, index) => toolName === normalizedIncoming[index]) + } +} diff --git a/src/services/self-improving/ReviewPromptFactory.ts b/src/services/self-improving/ReviewPromptFactory.ts new file mode 100644 index 0000000000..016efe1ec9 --- /dev/null +++ b/src/services/self-improving/ReviewPromptFactory.ts @@ -0,0 +1,111 @@ +/** + * Review type + */ +export type ReviewType = "memory" | "skill" | "combined" + +/** + * Review prompt result + */ +export interface ReviewPrompt { + type: ReviewType + systemPrompt: string + userPrompt: string +} + +/** + * ReviewPromptFactory — generates structured review prompts. + */ +export class ReviewPromptFactory { + createMemoryReviewPrompt(transcriptSummary: string): ReviewPrompt { + return { + type: "memory", + systemPrompt: `You are a memory review specialist. Your task is to review the recent conversation transcript and identify durable facts that should be saved to long-term memory. + +## Guidelines +- Save facts that are likely to be useful across multiple sessions +- Save user preferences, project conventions, and environment details +- Do NOT save transient information (one-off commands, temporary errors) +- Do NOT save information that is already in memory +- Prefer concise, actionable facts over verbose descriptions +- Each fact should be a single, clear statement + +## Output Format +For each fact you want to save, output: +FACT: +CATEGORY: +REASON: `, + userPrompt: `Review the following conversation transcript and identify durable facts to save to memory. + +${transcriptSummary} + +Output your recommendations following the specified format.`, + } + } + + createSkillReviewPrompt(transcriptSummary: string): ReviewPrompt { + return { + type: "skill", + systemPrompt: `You are a skill review specialist. Your task is to review the recent conversation transcript and identify reusable procedures that should be saved as skills. + +## Guidelines +- Create skills for procedures that are repeated or likely to be repeated +- Update existing skills if the transcript reveals improvements +- Prefer class-level skills over one-off task narratives +- Avoid creating skills for transient environment failures +- Each skill should have a clear, single responsibility +- Support files (scripts, templates) should be separate from the main skill definition + +## Priority Order +1. Update an existing loaded skill if the transcript reveals improvements +2. Create an umbrella skill that groups related procedures +3. Add a support file (script, template) to an existing skill +4. Create a new standalone skill + +## Output Format +For each skill action, output: +ACTION: +SKILL_NAME: +DESCRIPTION: +CONTENT: +REASON: `, + userPrompt: `Review the following conversation transcript and identify reusable procedures to save as skills. + +${transcriptSummary} + +Output your recommendations following the specified format.`, + } + } + + createCombinedReviewPrompt(transcriptSummary: string): ReviewPrompt { + return { + type: "combined", + systemPrompt: `You are a self-improvement review specialist. Your task is to review the recent conversation transcript and identify both durable facts (memory) and reusable procedures (skills). + +## Memory Guidelines +- Memory is for facts: user preferences, project conventions, environment details +- Save facts that are durable and useful across sessions +- Each fact should be concise and actionable + +## Skill Guidelines +- Skills are for procedures: repeatable workflows, command sequences, code patterns +- Create skills for procedures likely to be repeated +- Update existing skills with improvements from the transcript +- Prefer class-level skills over one-off narratives + +## Output Format +For memory facts: +MEMORY_FACT: +MEMORY_CATEGORY: + +For skill actions: +SKILL_ACTION: +SKILL_NAME: +SKILL_CONTENT: `, + userPrompt: `Review the following conversation transcript and identify both durable facts (memory) and reusable procedures (skills). + +${transcriptSummary} + +Output your recommendations following the specified format.`, + } + } +} diff --git a/src/services/self-improving/SelfImprovingManager.ts b/src/services/self-improving/SelfImprovingManager.ts new file mode 100644 index 0000000000..b79247d74f --- /dev/null +++ b/src/services/self-improving/SelfImprovingManager.ts @@ -0,0 +1,735 @@ +import path from "path" +import crypto from "crypto" + +import type { + CodeIndexInfo, + Experiments, + ImprovementAction, + LearnedPattern, + LearningEvent, + Logger, + PromptContext, + SelfImprovingManagerOptions, + SelfImprovingScope, + TaskEventInfo, +} from "./types" +import { LearningStore } from "./LearningStore" +import { FeedbackCollector } from "./FeedbackCollector" +import { PatternAnalyzer } from "./PatternAnalyzer" +import { ImprovementApplier } from "./ImprovementApplier" +import { CodeIndexAdapter } from "./CodeIndexAdapter" +import type { MemoryBackend } from "./MemoryBackend" +import { MemoryBackendFactory } from "./MemoryBackendFactory" +import { MemoryStore } from "./MemoryStore" +import { SkillUsageStore } from "./SkillUsageStore" +import { ActionExecutor } from "./ActionExecutor" +import { CuratorService } from "./CuratorService" +import type { CuratorReport } from "./CuratorService" +import { ReviewPromptFactory } from "./ReviewPromptFactory" +import { TranscriptRecall } from "./TranscriptRecall" + +const SELF_IMPROVING_EXPERIMENT_ID = "selfImproving" +const REVIEW_CHECK_INTERVAL_MS = 60_000 + +type Runtime = { + store: LearningStore + feedbackCollector: FeedbackCollector + patternAnalyzer: PatternAnalyzer + improvementApplier: ImprovementApplier + codeIndexAdapter: CodeIndexAdapter +} + +export class SelfImprovingManager { + private readonly globalStoragePath: string + private readonly logger: Logger + private readonly getExperiments: () => Experiments | undefined + private readonly getCodeIndexInfo: SelfImprovingManagerOptions["getCodeIndexInfo"] + private readonly getMemoryBackend: SelfImprovingManagerOptions["getMemoryBackend"] + private readonly getAgentMemoryUrl: SelfImprovingManagerOptions["getAgentMemoryUrl"] + private readonly getSelfImprovingScope: SelfImprovingManagerOptions["getSelfImprovingScope"] + private readonly getAutoSkillsScope: SelfImprovingManagerOptions["getAutoSkillsScope"] + private readonly getWorkspacePath: SelfImprovingManagerOptions["getWorkspacePath"] + private readonly curatorConfig: SelfImprovingManagerOptions["curatorConfig"] + private readonly skillsManager: SelfImprovingManagerOptions["skillsManager"] + public memoryStore: MemoryBackend + public skillUsageStore: SkillUsageStore + public curatorService: CuratorService + public readonly reviewPromptFactory: ReviewPromptFactory + public transcriptRecall: TranscriptRecall + private actionExecutor: ActionExecutor + private memoryBackendType: "builtin" | "agentmemory" + private agentMemoryUrl: string | undefined + private storageBasePath: string + + private runtime: Runtime | undefined + private started = false + private reviewTimer: ReturnType | null = null + private curatorTimer: ReturnType | null = null + private promptRevision = 0 + private lastUserActivityAt = 0 + private reviewInFlight = false + private curatorInFlight = false + + constructor(options: SelfImprovingManagerOptions) { + this.globalStoragePath = options.globalStoragePath + this.logger = options.logger + this.getExperiments = options.getExperiments + this.getCodeIndexInfo = options.getCodeIndexInfo + this.getMemoryBackend = options.getMemoryBackend + this.getAgentMemoryUrl = options.getAgentMemoryUrl + this.getSelfImprovingScope = options.getSelfImprovingScope + this.getAutoSkillsScope = options.getAutoSkillsScope + this.getWorkspacePath = options.getWorkspacePath + this.curatorConfig = options.curatorConfig + this.skillsManager = options.skillsManager + this.memoryBackendType = this.resolveMemoryBackend(options.memoryBackend) + this.agentMemoryUrl = this.resolveAgentMemoryUrl(options.agentMemoryUrl) + this.storageBasePath = this.resolveStorageBasePath() + this.memoryStore = this.createMemoryStore() + this.skillUsageStore = this.createSkillUsageStore() + this.actionExecutor = this.createActionExecutor() + this.curatorService = this.createCuratorService() + this.reviewPromptFactory = new ReviewPromptFactory() + this.transcriptRecall = this.createTranscriptRecall() + } + + static isExperimentEnabled(experiments: Experiments | undefined): boolean { + if (!experiments) { + return false + } + + return experiments[SELF_IMPROVING_EXPERIMENT_ID] === true + } + + static isAutoSkillsEnabled(experiments: Experiments | undefined): boolean { + if (!SelfImprovingManager.isExperimentEnabled(experiments)) { + return false + } + + return experiments?.selfImprovingAutoSkills === true + } + + async initialize(): Promise { + if (!SelfImprovingManager.isExperimentEnabled(this.getExperiments())) { + return + } + + if (this.started) { + return + } + + try { + const runtime = this.getOrCreateRuntime() + await runtime.store.initialize() + await this.memoryStore.initialize() + await this.skillUsageStore.initialize() + await this.transcriptRecall.initialize() + await this.curatorService.initialize() + this.started = true + this.startTimers(runtime.store) + this.logger.appendLine( + "[SelfImprovingManager] Initialized: " + + `${runtime.store.getPatterns().length} patterns, ` + + `${runtime.store.getRecentEvents().length} events`, + ) + } catch (error) { + this.stopTimers() + this.started = false + this.runtime = undefined + this.logError("Initialization error", error) + } + } + + async handleExperimentChange(enabled?: boolean): Promise { + try { + const experimentEnabled = SelfImprovingManager.isExperimentEnabled(this.getExperiments()) + const shouldEnable = enabled ?? experimentEnabled + if (!shouldEnable || !experimentEnabled) { + await this.dispose() + return + } + + await this.initialize() + } catch (error) { + this.logError("Experiment change handling error", error) + } + } + + /** + * Handle settings change — called when experiments are updated. + * This enables/disables the module at runtime. + */ + async onSettingsChanged(_experiments: Experiments | undefined): Promise { + await this.reconfigureIfNeeded() + await this.handleExperimentChange() + } + + async dispose(): Promise { + const enabled = SelfImprovingManager.isExperimentEnabled(this.getExperiments()) + if (!enabled && !this.started) { + return + } + + this.stopTimers() + + try { + if (this.started) { + await this.runtime?.store.persist() + if (this.memoryStore instanceof MemoryStore) { + this.memoryStore.takeSnapshot() + } + } + + await this.memoryStore.dispose() + } catch (error) { + this.logError("Persist on dispose error", error) + } finally { + this.started = false + this.promptRevision = 0 + this.runtime = undefined + } + } + + async recordTaskCompletion(info: TaskEventInfo): Promise { + if (!SelfImprovingManager.isExperimentEnabled(this.getExperiments())) { + return + } + + if (!this.started || !this.runtime) { + return + } + + try { + const event = this.runtime.feedbackCollector.createTaskEvent(info) + this.runtime.store.addEvent(event) + this.lastUserActivityAt = event.timestamp + await this.transcriptRecall.record({ + id: event.id, + timestamp: event.timestamp, + taskId: info.taskId, + mode: info.mode, + summary: info.success + ? `Task completed: ${info.mode || "unknown"}` + : `Task failed: ${info.errorKey || "unknown"}`, + signal: info.success ? "TASK_SUCCESS" : "TASK_FAILURE", + workspacePath: info.workspacePath, + toolNames: info.toolNames, + errorKey: info.errorKey, + success: info.success, + }) + this.runtime.store.incrementToolIterations( + Math.max(1, info.toolIterationCount ?? info.toolNames?.length ?? 1), + ) + await this.checkReviewTriggers(this.runtime.store) + } catch (error) { + this.logError("recordTaskCompletion error", error) + } + } + + async recordUserCorrection(info: TaskEventInfo): Promise { + if (!SelfImprovingManager.isExperimentEnabled(this.getExperiments())) { + return + } + + if (!this.started || !this.runtime) { + return + } + + try { + const event = this.runtime.feedbackCollector.createCorrectionEvent(info) + this.runtime.store.addEvent(event) + this.lastUserActivityAt = event.timestamp + await this.checkReviewTriggers(this.runtime.store) + } catch (error) { + this.logError("recordUserCorrection error", error) + } + } + + async recordUserTurn(): Promise { + if (!SelfImprovingManager.isExperimentEnabled(this.getExperiments())) { + return + } + + if (!this.started || !this.runtime) { + return + } + + try { + this.lastUserActivityAt = Date.now() + this.runtime.store.incrementUserTurns() + await this.checkReviewTriggers(this.runtime.store) + } catch (error) { + this.logError("recordUserTurn error", error) + } + } + + async recordCodeIndexEvent(taskId?: string, codeIndexInfo?: CodeIndexInfo): Promise { + if (!SelfImprovingManager.isExperimentEnabled(this.getExperiments())) { + return + } + + if (!this.started || !this.runtime) { + return + } + + try { + if (!this.runtime.store.getConfig().codeIndexCorrelationEnabled) { + return + } + + const resolvedCodeIndexInfo = codeIndexInfo ?? this.runtime.codeIndexAdapter.getInfo() + const event = this.runtime.feedbackCollector.createCodeIndexEvent(resolvedCodeIndexInfo, taskId) + this.runtime.store.addEvent(event) + this.lastUserActivityAt = event.timestamp + } catch (error) { + this.logError("recordCodeIndexEvent error", error) + } + } + + async runReviewCycle(): Promise { + if (!SelfImprovingManager.isExperimentEnabled(this.getExperiments())) { + return + } + + if (!this.started || !this.runtime) { + return + } + + if (this.reviewInFlight) { + this.logger.appendLine("[SelfImprovingManager] Review cycle already in progress, skipping") + return + } + this.reviewInFlight = true + + try { + const events = [...this.runtime.store.getRecentEvents()] as LearningEvent[] + if (events.length === 0) { + return + } + + const existingPatterns = [...this.runtime.store.getPatterns()] as LearnedPattern[] + const newPatterns = this.runtime.patternAnalyzer.analyze(events, existingPatterns) + for (const pattern of newPatterns) { + this.runtime.store.addPattern(pattern) + } + + const actions = this.runtime.improvementApplier.generateActions([ + ...this.runtime.store.getPatterns(), + ] as LearnedPattern[]) + for (const action of actions) { + this.runtime.store.addAction(action) + } + + const pendingActions = [...this.runtime.store.getPendingActions()] as ImprovementAction[] + if (pendingActions.length > 0) { + const succeeded = await this.actionExecutor.executeBatch(pendingActions) + for (const actionId of succeeded) { + this.runtime.store.removeAction(actionId) + } + + this.logger.appendLine( + `[SelfImprovingManager] Executed ${succeeded.size}/${pendingActions.length} actions`, + ) + } + + this.updateReviewTelemetry(this.runtime.store, actions) + this.promptRevision += 1 + this.runtime.store.resetCounters() + await this.runtime.store.persist() + this.logger.appendLine( + `[SelfImprovingManager] Review cycle: ${newPatterns.length} patterns, ${actions.length} actions`, + ) + } catch (error) { + this.logError("Review cycle error", error) + } finally { + this.reviewInFlight = false + } + } + + async runCuratorCycle(): Promise { + if (!SelfImprovingManager.isExperimentEnabled(this.getExperiments())) { + return undefined + } + + if (!this.started || !this.runtime) { + return undefined + } + + if (this.curatorInFlight) { + return undefined + } + this.curatorInFlight = true + + try { + const now = Date.now() + const report = await this.curatorService.run( + now, + this.lastUserActivityAt > 0 ? this.lastUserActivityAt : undefined, + ) + this.runtime.store.updateTelemetry({ lastCuratorRunAt: report.timestamp }) + + if (report.transitions.length > 0) { + this.logger.appendLine(`[SelfImprovingManager] Curator cycle: ${report.transitions.length} transitions`) + } + + return report + } catch (error) { + this.logger.appendLine( + `[SelfImprovingManager] Curator cycle error: ${error instanceof Error ? error.message : String(error)}`, + ) + return undefined + } finally { + this.curatorInFlight = false + } + } + + getPromptContext(): PromptContext | undefined { + if (!SelfImprovingManager.isExperimentEnabled(this.getExperiments())) { + return undefined + } + + if (!this.started || !this.runtime) { + return undefined + } + + try { + return this.runtime.improvementApplier.getPromptContext( + [...this.runtime.store.getPatterns()] as LearnedPattern[], + this.runtime.store.getConfig().maxPromptPatterns, + this.promptRevision, + ) + } catch (error) { + this.logError("getPromptContext error", error) + return undefined + } + } + + getPromptContextString(): string { + if (!this.started) { + return "" + } + + try { + if (this.memoryStore instanceof MemoryStore) { + return this.memoryStore.getSnapshotString() + } + + return "" + } catch { + return "" + } + } + + async getStatus(): Promise<{ + enabled: boolean + started: boolean + patternCount: number + eventCount: number + actionCount: number + memoryEntries: number + memoryBackend?: string + skillRecords: number + curatorStatus: ReturnType + lastReviewAt?: number + lastCuratorRunAt?: number + }> { + const enabled = SelfImprovingManager.isExperimentEnabled(this.getExperiments()) + const curatorStatus = this.curatorService.getStatus() + if (!enabled) { + return { + enabled: false, + started: false, + patternCount: 0, + eventCount: 0, + actionCount: 0, + memoryEntries: 0, + skillRecords: 0, + curatorStatus, + } + } + + if (!this.started || !this.runtime) { + return { + enabled: true, + started: false, + patternCount: 0, + eventCount: 0, + actionCount: 0, + memoryEntries: 0, + skillRecords: 0, + curatorStatus, + } + } + + try { + const telemetry = this.runtime.store.getTelemetry() + const memStats = await this.memoryStore.getStats() + const skillStats = this.skillUsageStore.getStats() + return { + enabled: true, + started: true, + patternCount: this.runtime.store.getPatterns().length, + eventCount: this.runtime.store.getRecentEvents().length, + actionCount: this.runtime.store.getPendingActions().length, + memoryEntries: memStats.entryCount, + memoryBackend: memStats.backend, + skillRecords: skillStats.total, + curatorStatus, + lastReviewAt: telemetry.lastReviewAt, + lastCuratorRunAt: telemetry.lastCuratorRunAt, + } + } catch { + return { + enabled: true, + started: true, + patternCount: 0, + eventCount: 0, + actionCount: 0, + memoryEntries: 0, + skillRecords: 0, + curatorStatus, + } + } + } + + async reset(): Promise { + if (!SelfImprovingManager.isExperimentEnabled(this.getExperiments())) { + return + } + + if (!this.started || !this.runtime) { + return + } + + try { + await this.runtime.store.reset() + this.promptRevision = 0 + this.logger.appendLine("[SelfImprovingManager] Learning state reset") + } catch (error) { + this.logError("Reset error", error) + } + } + + private getOrCreateRuntime(): Runtime { + if (!this.runtime) { + this.runtime = { + store: new LearningStore(this.storageBasePath, this.logger), + feedbackCollector: new FeedbackCollector(), + patternAnalyzer: new PatternAnalyzer(), + improvementApplier: new ImprovementApplier({ + getSkillNames: () => this.skillsManager?.getSkillNames() ?? [], + getSkillProvenance: (name: string) => this.resolveSkillProvenance(name), + getSkillProvenanceForSource: (name: string, source: "global" | "project") => + this.resolveSkillProvenance(name, source), + hasSkill: (name: string, source: "global" | "project") => + this.skillsManager?.hasSkill?.(name, source) ?? false, + isAutoSkillsEnabled: () => SelfImprovingManager.isAutoSkillsEnabled(this.getExperiments()), + getAutoSkillsScope: () => this.resolveAutoSkillsScope(), + }), + codeIndexAdapter: new CodeIndexAdapter(this.logger, this.getCodeIndexInfo), + } + } + + return this.runtime + } + + private resolveMemoryBackend(fallback?: "builtin" | "agentmemory"): "builtin" | "agentmemory" { + return this.getMemoryBackend?.() ?? fallback ?? "builtin" + } + + private resolveAgentMemoryUrl(fallback?: string): string | undefined { + return this.getAgentMemoryUrl?.() ?? fallback + } + + private resolveSelfImprovingScope(): SelfImprovingScope { + return this.getSelfImprovingScope?.() ?? "global" + } + + private resolveAutoSkillsScope(): SelfImprovingScope { + return this.getAutoSkillsScope?.() ?? "workspace" + } + + private resolveStorageBasePath(): string { + if (this.resolveSelfImprovingScope() !== "workspace") { + return this.globalStoragePath + } + + const workspacePath = this.getWorkspacePath?.() + if (!workspacePath) { + return path.join(this.globalStoragePath, "workspace-scopes", "no-workspace") + } + + const workspaceHash = crypto.createHash("sha256").update(workspacePath).digest("hex").slice(0, 16) + return path.join(this.globalStoragePath, "workspace-scopes", workspaceHash) + } + + private createMemoryStore(): MemoryBackend { + return MemoryBackendFactory.create( + this.memoryBackendType, + this.storageBasePath, + this.logger, + this.agentMemoryUrl, + ) + } + + private createSkillUsageStore(): SkillUsageStore { + return new SkillUsageStore(this.storageBasePath, this.logger) + } + + private createCuratorService( + config: SelfImprovingManagerOptions["curatorConfig"] = this.curatorConfig, + ): CuratorService { + return new CuratorService(this.storageBasePath, this.skillUsageStore, this.logger, config) + } + + private createTranscriptRecall(): TranscriptRecall { + return new TranscriptRecall(this.storageBasePath, this.logger) + } + + private createActionExecutor(): ActionExecutor { + return new ActionExecutor(this.memoryStore, this.skillUsageStore, this.logger, this.skillsManager) + } + + private async reconfigureIfNeeded(): Promise { + const nextBackend = this.resolveMemoryBackend(this.memoryBackendType) + const nextUrl = this.resolveAgentMemoryUrl(this.agentMemoryUrl) + const nextStorageBasePath = this.resolveStorageBasePath() + const backendChanged = nextBackend !== this.memoryBackendType || nextUrl !== this.agentMemoryUrl + const storageChanged = nextStorageBasePath !== this.storageBasePath + + if (!backendChanged && !storageChanged) { + return + } + + const shouldRestart = this.started && SelfImprovingManager.isExperimentEnabled(this.getExperiments()) + if (shouldRestart) { + this.stopTimers() + await this.runtime?.store.persist() + if (this.memoryStore instanceof MemoryStore) { + this.memoryStore.takeSnapshot() + } + } + + await this.memoryStore.dispose() + this.started = false + this.runtime = undefined + this.promptRevision = 0 + + this.memoryBackendType = nextBackend + this.agentMemoryUrl = nextUrl + this.storageBasePath = nextStorageBasePath + this.memoryStore = this.createMemoryStore() + this.skillUsageStore = this.createSkillUsageStore() + this.actionExecutor = this.createActionExecutor() + this.curatorService = this.createCuratorService() + this.transcriptRecall = this.createTranscriptRecall() + + if (shouldRestart) { + await this.initialize() + } + + if (backendChanged) { + this.logger.appendLine( + `[SelfImprovingManager] Memory backend configured: ${this.memoryBackendType}${this.agentMemoryUrl ? ` (${this.agentMemoryUrl})` : ""}`, + ) + } + } + + private startTimers(store: LearningStore): void { + this.stopTimers() + const config = store.getConfig() + this.reviewTimer = setInterval(() => { + void this.runReviewCycle() + }, REVIEW_CHECK_INTERVAL_MS) + + if (config.curatorEnabled) { + this.curatorTimer = setInterval(() => { + this.runCuratorCycle().catch((error) => { + this.logger.appendLine( + `[SelfImprovingManager] Curator cycle error: ${error instanceof Error ? error.message : String(error)}`, + ) + }) + }, config.curatorIntervalMs) + } + } + + private stopTimers(): void { + if (this.reviewTimer) { + clearInterval(this.reviewTimer) + this.reviewTimer = null + } + + if (this.curatorTimer) { + clearInterval(this.curatorTimer) + this.curatorTimer = null + } + } + + private async checkReviewTriggers(store: LearningStore): Promise { + const counters = store.getCounters() + const config = store.getConfig() + if ( + counters.userTurnsSinceReview >= config.reviewOnTurnCount || + counters.toolIterationsSinceReview >= config.reviewOnToolIterationCount + ) { + await this.runReviewCycle() + } + } + + private updateReviewTelemetry(store: LearningStore, actions: ImprovementAction[]): void { + const telemetry = store.getTelemetry() + store.updateTelemetry({ + lastReviewAt: Date.now(), + promptEnrichmentUses: + telemetry.promptEnrichmentUses + + actions.filter((action) => action.actionType === "PROMPT_ENRICHMENT").length, + toolPreferenceUses: + telemetry.toolPreferenceUses + + actions.filter((action) => action.actionType === "TOOL_PREFERENCE").length, + errorAvoidanceUses: + telemetry.errorAvoidanceUses + + actions.filter((action) => action.actionType === "ERROR_AVOIDANCE").length, + skillSuggestionCount: + telemetry.skillSuggestionCount + + actions.filter( + (action) => + action.actionType === "SKILL_SUGGESTION" || + action.actionType === "SKILL_CREATE" || + action.actionType === "SKILL_UPDATE", + ).length, + }) + } + + private resolveSkillProvenance(name: string, source?: "global" | "project"): string { + const agentRecord = this.skillUsageStore.getAll().find((record) => { + if (record.createdBy !== "agent" || record.skillName !== name) { + return false + } + + if (!source) { + return true + } + + return record.skillId === this.buildSkillId(name, source) + }) + if (agentRecord) { + return agentRecord.createdBy + } + + if (source) { + return this.skillsManager?.getSkillProvenanceForSource?.(name, source) ?? "unknown" + } + + return this.skillsManager?.getSkillProvenance(name) ?? "unknown" + } + + private buildSkillId(name: string, source: "global" | "project"): string { + return `skill:${source}:${name}` + } + + private logError(context: string, error: unknown): void { + this.logger.appendLine( + `[SelfImprovingManager] ${context}: ${error instanceof Error ? error.message : String(error)}`, + ) + } +} diff --git a/src/services/self-improving/SkillUsageStore.ts b/src/services/self-improving/SkillUsageStore.ts new file mode 100644 index 0000000000..7554a2a2da --- /dev/null +++ b/src/services/self-improving/SkillUsageStore.ts @@ -0,0 +1,426 @@ +import * as fs from "fs/promises" +import * as path from "path" + +import { safeWriteJson } from "../../utils/safeWriteJson" +import type { Logger } from "./types" + +/** + * Skill provenance - who created the skill + */ +export type SkillProvenance = "agent" | "user" | "bundled" | "hub" | "unknown" + +/** + * Skill lifecycle state + */ +export type SkillLifecycleState = "active" | "stale" | "archived" + +/** + * Skill telemetry record + */ +export interface SkillTelemetryRecord { + /** Unique skill identifier */ + skillId: string + /** Skill name */ + skillName: string + /** Who created this skill */ + createdBy: SkillProvenance + /** Current lifecycle state */ + state: SkillLifecycleState + /** Whether the skill is pinned (protected from auto-mutation) */ + pinned: boolean + /** Number of times the skill has been loaded/viewed */ + viewCount: number + /** Number of times the skill has been used in a task */ + useCount: number + /** Number of times the skill has been patched/updated */ + patchCount: number + /** Timestamp of first creation */ + createdAt: number + /** Timestamp of last activity */ + lastActivityAt: number + /** Timestamp of archival (if archived) */ + archivedAt?: number + /** Tags for categorization */ + tags?: string[] +} + +const SKILL_PROVENANCE_VALUES: ReadonlySet = new Set(["agent", "user", "bundled", "hub", "unknown"]) +const SKILL_LIFECYCLE_VALUES: ReadonlySet = new Set(["active", "stale", "archived"]) + +/** + * SkillUsageStore - telemetry sidecar for skill usage tracking. + * + * Mirrors Hermes' skill_usage.py pattern with: + * - Use/view/patch counters + * - Provenance tracking (agent-created vs user-authored vs bundled) + * - Pinning support (protected from autonomous mutation) + * - Lifecycle state management + * - Atomic file persistence + */ +export class SkillUsageStore { + private readonly filePath: string + private readonly logger: Logger + private records: Map = new Map() + private initialized = false + + constructor(baseDir: string, logger: Logger) { + this.filePath = path.join(baseDir, "self-improving", "skill-usage.json") + this.logger = logger + } + + /** + * Initialize the store - load persisted telemetry from disk. + */ + async initialize(): Promise { + if (this.initialized) { + return + } + + try { + await fs.mkdir(path.dirname(this.filePath), { recursive: true }) + await this.loadFromDisk() + this.logger.appendLine(`[SkillUsageStore] Initialized: ${this.records.size} skill records`) + } catch (error) { + this.logger.appendLine( + `[SkillUsageStore] Initialization error: ${error instanceof Error ? error.message : String(error)}`, + ) + } finally { + this.initialized = true + } + } + + /** + * Load telemetry from disk. + */ + private async loadFromDisk(): Promise { + try { + const raw = await fs.readFile(this.filePath, "utf-8") + const parsed = JSON.parse(raw) + + if (!Array.isArray(parsed)) { + return + } + + for (const candidate of parsed) { + const record = this.sanitizeRecord(candidate) + if (record) { + this.records.set(record.skillId, record) + } + } + } catch (error: unknown) { + const errorCode = typeof error === "object" && error !== null && "code" in error ? error.code : undefined + if (errorCode !== "ENOENT") { + this.logger.appendLine( + `[SkillUsageStore] Load error: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + } + + /** + * Persist telemetry to disk atomically. + */ + private async persist(): Promise { + try { + await safeWriteJson(this.filePath, Array.from(this.records.values()), { prettyPrint: true }) + } catch (error) { + this.logger.appendLine( + `[SkillUsageStore] Persist error: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + + private queuePersist(): void { + void this.persist() + } + + private getRecord(skillId: string): SkillTelemetryRecord | undefined { + return this.records.get(skillId) + } + + private cloneRecord(record: SkillTelemetryRecord): SkillTelemetryRecord { + return { + ...record, + tags: record.tags ? [...record.tags] : undefined, + } + } + + private sanitizeRecord(value: unknown): SkillTelemetryRecord | null { + if (!value || typeof value !== "object") { + return null + } + + const candidate = value as Partial + if (typeof candidate.skillId !== "string" || candidate.skillId.trim().length === 0) { + return null + } + + const now = Date.now() + const createdAt = typeof candidate.createdAt === "number" ? candidate.createdAt : now + const lastActivityAt = typeof candidate.lastActivityAt === "number" ? candidate.lastActivityAt : createdAt + + return { + skillId: candidate.skillId, + skillName: + typeof candidate.skillName === "string" && candidate.skillName.trim().length > 0 + ? candidate.skillName + : candidate.skillId, + createdBy: this.normalizeProvenance(candidate.createdBy), + state: this.normalizeState(candidate.state), + pinned: candidate.pinned === true, + viewCount: this.normalizeCounter(candidate.viewCount), + useCount: this.normalizeCounter(candidate.useCount), + patchCount: this.normalizeCounter(candidate.patchCount), + createdAt, + lastActivityAt, + archivedAt: typeof candidate.archivedAt === "number" ? candidate.archivedAt : undefined, + tags: this.normalizeTags(candidate.tags), + } + } + + private normalizeCounter(value: number | undefined): number { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : 0 + } + + private normalizeProvenance(value: SkillProvenance | undefined): SkillProvenance { + return value && SKILL_PROVENANCE_VALUES.has(value) ? value : "unknown" + } + + private normalizeState(value: SkillLifecycleState | undefined): SkillLifecycleState { + return value && SKILL_LIFECYCLE_VALUES.has(value) ? value : "active" + } + + private normalizeTags(tags: string[] | undefined): string[] | undefined { + if (!Array.isArray(tags)) { + return undefined + } + + const normalized = Array.from(new Set(tags.map((tag) => tag.trim()).filter((tag) => tag.length > 0))) + + return normalized.length > 0 ? normalized : undefined + } + + // ──── Record management ──── + + /** + * Get or create a telemetry record for a skill. + */ + getOrCreate(skillId: string, skillName: string, createdBy: SkillProvenance = "unknown"): SkillTelemetryRecord { + const existing = this.getRecord(skillId) + if (existing) { + return this.cloneRecord(existing) + } + + const now = Date.now() + const record: SkillTelemetryRecord = { + skillId, + skillName, + createdBy, + state: "active", + pinned: false, + viewCount: 0, + useCount: 0, + patchCount: 0, + createdAt: now, + lastActivityAt: now, + } + + this.records.set(skillId, record) + this.queuePersist() + + return this.cloneRecord(record) + } + + /** + * Get a telemetry record by skill ID. + */ + get(skillId: string): SkillTelemetryRecord | undefined { + const record = this.getRecord(skillId) + return record ? this.cloneRecord(record) : undefined + } + + /** + * Get all telemetry records. + */ + getAll(): SkillTelemetryRecord[] { + return Array.from(this.records.values(), (record) => this.cloneRecord(record)) + } + + /** + * Get records filtered by provenance. + */ + getByProvenance(provenance: SkillProvenance): SkillTelemetryRecord[] { + return this.getAll().filter((record) => record.createdBy === provenance) + } + + /** + * Get records filtered by lifecycle state. + */ + getByState(state: SkillLifecycleState): SkillTelemetryRecord[] { + return this.getAll().filter((record) => record.state === state) + } + + // ──── Telemetry bumps ──── + + /** + * Record that a skill was viewed/loaded. + */ + async bumpView(skillId: string): Promise { + const record = this.getRecord(skillId) + if (!record) { + return + } + + record.viewCount += 1 + record.lastActivityAt = Date.now() + await this.persist() + } + + /** + * Record that a skill was used in a task. + */ + async bumpUse(skillId: string): Promise { + const record = this.getRecord(skillId) + if (!record) { + return + } + + record.useCount += 1 + record.lastActivityAt = Date.now() + await this.persist() + } + + /** + * Record that a skill was patched/updated. + */ + async bumpPatch(skillId: string): Promise { + const record = this.getRecord(skillId) + if (!record) { + return + } + + record.patchCount += 1 + record.lastActivityAt = Date.now() + await this.persist() + } + + // ──── Lifecycle management ──── + + /** + * Pin a skill (protect from autonomous mutation). + */ + async pin(skillId: string): Promise { + const record = this.getRecord(skillId) + if (!record) { + return + } + + record.pinned = true + record.lastActivityAt = Date.now() + await this.persist() + } + + /** + * Unpin a skill. + */ + async unpin(skillId: string): Promise { + const record = this.getRecord(skillId) + if (!record) { + return + } + + record.pinned = false + record.lastActivityAt = Date.now() + await this.persist() + } + + /** + * Check if a skill is pinned. + */ + isPinned(skillId: string): boolean { + return this.getRecord(skillId)?.pinned ?? false + } + + /** + * Transition a skill to a new lifecycle state. + */ + async transitionState(skillId: string, newState: SkillLifecycleState): Promise { + const record = this.getRecord(skillId) + if (!record || record.pinned) { + return + } + + record.state = newState + record.lastActivityAt = Date.now() + + if (newState === "archived") { + record.archivedAt = Date.now() + } else { + delete record.archivedAt + } + + await this.persist() + } + + /** + * Get skills eligible for stale transition. + * Skills with no activity for staleAfterDays. + */ + getStaleCandidates(staleAfterDays: number): SkillTelemetryRecord[] { + const threshold = Date.now() - staleAfterDays * 24 * 60 * 60 * 1000 + return this.getAll().filter( + (record) => record.state === "active" && !record.pinned && record.lastActivityAt < threshold, + ) + } + + /** + * Get skills eligible for archive transition. + */ + getArchiveCandidates(archiveAfterDays: number): SkillTelemetryRecord[] { + const threshold = Date.now() - archiveAfterDays * 24 * 60 * 60 * 1000 + return this.getAll().filter( + (record) => record.state === "stale" && !record.pinned && record.lastActivityAt < threshold, + ) + } + + /** + * Remove a skill record entirely. + */ + async remove(skillId: string): Promise { + if (!this.records.delete(skillId)) { + return + } + + await this.persist() + } + + /** + * Get aggregate statistics. + */ + getStats(): { + total: number + active: number + stale: number + archived: number + pinned: number + agentCreated: number + } { + const records = Array.from(this.records.values()) + return { + total: records.length, + active: records.filter((record) => record.state === "active").length, + stale: records.filter((record) => record.state === "stale").length, + archived: records.filter((record) => record.state === "archived").length, + pinned: records.filter((record) => record.pinned).length, + agentCreated: records.filter((record) => record.createdBy === "agent").length, + } + } + + /** + * Reset all telemetry. + */ + async reset(): Promise { + this.records.clear() + await this.persist() + } +} diff --git a/src/services/self-improving/TranscriptRecall.ts b/src/services/self-improving/TranscriptRecall.ts new file mode 100644 index 0000000000..92c36c8061 --- /dev/null +++ b/src/services/self-improving/TranscriptRecall.ts @@ -0,0 +1,189 @@ +import * as fs from "fs/promises" +import * as path from "path" + +import { safeWriteJson } from "../../utils/safeWriteJson" +import type { Logger } from "./types" + +/** + * Transcript entry — a single recorded event or task outcome + */ +export interface TranscriptEntry { + id: string + timestamp: number + taskId?: string + mode?: string + summary: string + signal: string + workspacePath?: string + toolNames?: string[] + errorKey?: string + success?: boolean +} + +/** + * TranscriptRecall — searchable transcript evidence store. + */ +export class TranscriptRecall { + private readonly filePath: string + private readonly logger: Logger + private entries: TranscriptEntry[] = [] + private initialized = false + private initializePromise: Promise | null = null + + private static readonly MAX_ENTRIES = 1000 + + constructor(baseDir: string, logger: Logger) { + this.filePath = path.join(baseDir, "self-improving", "transcript-recall.json") + this.logger = logger + } + + async initialize(): Promise { + if (this.initialized) { + return + } + + if (!this.initializePromise) { + this.initializePromise = (async () => { + try { + await fs.mkdir(path.dirname(this.filePath), { recursive: true }) + await this.loadFromDisk() + } catch (error) { + this.logger.appendLine( + `[TranscriptRecall] Initialization error: ${error instanceof Error ? error.message : String(error)}`, + ) + } finally { + this.initialized = true + this.initializePromise = null + } + })() + } + + await this.initializePromise + } + + async record(entry: TranscriptEntry): Promise { + if (!this.initialized) { + await this.initialize() + } + + this.entries.push({ + ...entry, + toolNames: entry.toolNames ? [...entry.toolNames] : undefined, + }) + + if (this.entries.length > TranscriptRecall.MAX_ENTRIES) { + this.entries = this.entries.slice(-TranscriptRecall.MAX_ENTRIES) + } + + await this.persist() + } + + search(query: string): TranscriptEntry[] { + const normalizedQuery = query.toLowerCase() + return this.entries + .filter((entry) => { + if (entry.summary.toLowerCase().includes(normalizedQuery)) { + return true + } + + if (entry.signal.toLowerCase().includes(normalizedQuery)) { + return true + } + + if (entry.errorKey?.toLowerCase().includes(normalizedQuery)) { + return true + } + + if (entry.mode?.toLowerCase().includes(normalizedQuery)) { + return true + } + + return entry.toolNames?.some((toolName) => toolName.toLowerCase().includes(normalizedQuery)) ?? false + }) + .slice(-20) + } + + searchBySignal(signal: string): TranscriptEntry[] { + return this.entries.filter((entry) => entry.signal === signal).slice(-50) + } + + searchByErrorKey(errorKey: string): TranscriptEntry[] { + return this.entries.filter((entry) => entry.errorKey === errorKey).slice(-20) + } + + getRecent(count = 10): TranscriptEntry[] { + return this.entries.slice(-count) + } + + get size(): number { + return this.entries.length + } + + async clear(): Promise { + this.entries = [] + await this.persist() + } + + private async loadFromDisk(): Promise { + try { + const raw = await fs.readFile(this.filePath, "utf-8") + const parsed = JSON.parse(raw) + if (Array.isArray(parsed)) { + this.entries = parsed + .map((entry) => this.sanitizeEntry(entry)) + .filter((entry): entry is TranscriptEntry => entry !== null) + .slice(-TranscriptRecall.MAX_ENTRIES) + } + } catch (error: unknown) { + const errorCode = typeof error === "object" && error !== null && "code" in error ? error.code : undefined + if (errorCode !== "ENOENT") { + this.logger.appendLine( + `[TranscriptRecall] Load error: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + } + + private sanitizeEntry(value: unknown): TranscriptEntry | null { + if (!value || typeof value !== "object") { + return null + } + + const candidate = value as Partial + if ( + typeof candidate.id !== "string" || + typeof candidate.timestamp !== "number" || + typeof candidate.summary !== "string" || + typeof candidate.signal !== "string" + ) { + return null + } + + return { + id: candidate.id, + timestamp: candidate.timestamp, + taskId: typeof candidate.taskId === "string" ? candidate.taskId : undefined, + mode: typeof candidate.mode === "string" ? candidate.mode : undefined, + summary: candidate.summary, + signal: candidate.signal, + workspacePath: typeof candidate.workspacePath === "string" ? candidate.workspacePath : undefined, + toolNames: + Array.isArray(candidate.toolNames) && + candidate.toolNames.every((toolName) => typeof toolName === "string") + ? [...candidate.toolNames] + : undefined, + errorKey: typeof candidate.errorKey === "string" ? candidate.errorKey : undefined, + success: typeof candidate.success === "boolean" ? candidate.success : undefined, + } + } + + private async persist(): Promise { + try { + await safeWriteJson(this.filePath, this.entries, { prettyPrint: true }) + } catch (error) { + this.logger.appendLine( + `[TranscriptRecall] Persist error: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } +} diff --git a/src/services/self-improving/__tests__/ActionExecutor.spec.ts b/src/services/self-improving/__tests__/ActionExecutor.spec.ts new file mode 100644 index 0000000000..974cc10093 --- /dev/null +++ b/src/services/self-improving/__tests__/ActionExecutor.spec.ts @@ -0,0 +1,178 @@ +import { ActionExecutor } from "../ActionExecutor" +import type { ImprovementAction } from "../types" + +describe("ActionExecutor", () => { + const logger = { appendLine: vi.fn() } + + beforeEach(() => { + logger.appendLine.mockReset() + }) + + it("writes prompt, error, and tool guidance into memory", async () => { + const memoryStore = { + store: vi.fn().mockResolvedValue({ id: "mem-1" }), + } as any + const skillUsageStore = { getOrCreate: vi.fn() } as any + const executor = new ActionExecutor(memoryStore, skillUsageStore, logger) + + const actions: ImprovementAction[] = [ + { + id: "action-1", + actionType: "PROMPT_ENRICHMENT", + target: "system-prompt", + payload: { summary: "Prefer semantic search before regex search" }, + timestamp: 1, + }, + { + id: "action-2", + actionType: "ERROR_AVOIDANCE", + target: "task-execution", + payload: { summary: "Handle ENOENT before retry", errorKeys: ["ENOENT"] }, + timestamp: 2, + }, + { + id: "action-3", + actionType: "TOOL_PREFERENCE", + target: "task-execution", + payload: { summary: "Use codebase_search before search_files", toolNames: ["codebase_search"] }, + timestamp: 3, + }, + ] + + const succeeded = await executor.executeBatch(actions) + + expect(succeeded).toEqual(new Set(["action-1", "action-2", "action-3"])) + expect(memoryStore.store).toHaveBeenNthCalledWith(1, { + content: "Prefer semantic search before regex search", + source: "learning", + tags: ["learned", "prompt"], + }) + expect(memoryStore.store).toHaveBeenNthCalledWith(2, { + content: "Handle ENOENT before retry", + source: "learning", + tags: ["error-avoidance", "error:ENOENT"], + }) + expect(memoryStore.store).toHaveBeenNthCalledWith(3, { + content: "Use codebase_search before search_files", + source: "learning", + tags: ["tool-preference", "tool:codebase_search"], + }) + }) + + it("records skill suggestions in the telemetry sidecar", async () => { + const memoryStore = { store: vi.fn() } as any + const skillUsageStore = { getOrCreate: vi.fn() } as any + const executor = new ActionExecutor(memoryStore, skillUsageStore, logger) + + const action: ImprovementAction = { + id: "action-skill", + actionType: "SKILL_SUGGESTION", + target: "skills-manager", + payload: { + summary: "Create a self-improving review skill", + skillName: "Self Improving Review Skill", + }, + timestamp: 1, + } + + await expect(executor.execute(action)).resolves.toBe(true) + expect(skillUsageStore.getOrCreate).toHaveBeenCalledWith( + expect.stringMatching(/^suggested:/), + "Self Improving Review Skill", + "agent", + ) + expect(logger.appendLine).toHaveBeenCalledWith( + "[ActionExecutor] Skill suggestion recorded: Create a self-improving review skill", + ) + }) + + it("creates agent-managed skills from mutation actions", async () => { + const memoryStore = { store: vi.fn() } as any + const skillUsageStore = { getOrCreate: vi.fn() } as any + const skillsManager = { + createSkillFromContent: vi + .fn() + .mockResolvedValue("/tmp/.roo/skills/workflow-read-file-search-files/SKILL.md"), + } as any + const executor = new ActionExecutor(memoryStore, skillUsageStore, logger, skillsManager) + + const action: ImprovementAction = { + id: "action-skill-create", + actionType: "SKILL_CREATE", + target: "skills-manager", + payload: { + skillName: "workflow-read-file-search-files", + description: "Use when tasks repeatedly succeed with read_file and search_files.", + content: + "---\nname: workflow-read-file-search-files\ndescription: Use when tasks repeatedly succeed with read_file and search_files.\n---\n\n# Workflow\n", + source: "project", + modeSlugs: ["code"], + }, + timestamp: 1, + } + + await expect(executor.execute(action)).resolves.toBe(true) + expect(skillsManager.createSkillFromContent).toHaveBeenCalledWith( + "workflow-read-file-search-files", + "project", + "Use when tasks repeatedly succeed with read_file and search_files.", + expect.stringContaining("name: workflow-read-file-search-files"), + ["code"], + ) + expect(skillUsageStore.getOrCreate).toHaveBeenCalledWith( + "skill:project:workflow-read-file-search-files", + "workflow-read-file-search-files", + "agent", + ) + }) + + it("updates existing agent-managed skills from mutation actions", async () => { + const memoryStore = { store: vi.fn() } as any + const skillUsageStore = { + getOrCreate: vi.fn(), + bumpPatch: vi.fn().mockResolvedValue(undefined), + } as any + const skillsManager = { + updateSkillContent: vi.fn().mockResolvedValue(undefined), + } as any + const executor = new ActionExecutor(memoryStore, skillUsageStore, logger, skillsManager) + + const action: ImprovementAction = { + id: "action-skill-update", + actionType: "SKILL_UPDATE", + target: "skills-manager", + payload: { + skillId: "skill:project:workflow-read-file-search-files", + skillName: "workflow-read-file-search-files", + content: + "---\nname: workflow-read-file-search-files\ndescription: Updated workflow\n---\n\n# Workflow\nUpdated\n", + source: "project", + mode: "code", + }, + timestamp: 1, + } + + await expect(executor.execute(action)).resolves.toBe(true) + expect(skillsManager.updateSkillContent).toHaveBeenCalledWith( + "workflow-read-file-search-files", + "project", + expect.stringContaining("Updated workflow"), + "code", + ) + expect(skillUsageStore.bumpPatch).toHaveBeenCalledWith("skill:project:workflow-read-file-search-files") + }) + + it("keeps invalid actions pending by reporting failure", async () => { + const executor = new ActionExecutor({ store: vi.fn() } as any, { getOrCreate: vi.fn() } as any, logger) + + await expect( + executor.execute({ + id: "invalid", + actionType: "PROMPT_ENRICHMENT", + target: "system-prompt", + payload: {}, + timestamp: 1, + }), + ).resolves.toBe(false) + }) +}) diff --git a/src/services/self-improving/__tests__/AgentMemoryAdapter.spec.ts b/src/services/self-improving/__tests__/AgentMemoryAdapter.spec.ts new file mode 100644 index 0000000000..b7b51a4dc3 --- /dev/null +++ b/src/services/self-improving/__tests__/AgentMemoryAdapter.spec.ts @@ -0,0 +1,127 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" + +import { AgentMemoryAdapter } from "../AgentMemoryAdapter" + +describe("AgentMemoryAdapter", () => { + const logger = { appendLine: vi.fn() } + const adapters: AgentMemoryAdapter[] = [] + + beforeEach(() => { + logger.appendLine.mockReset() + }) + + afterEach(async () => { + await Promise.all(adapters.splice(0).map((adapter) => adapter.dispose())) + vi.unstubAllGlobals() + vi.restoreAllMocks() + }) + + function createAdapter(): AgentMemoryAdapter { + const adapter = new AgentMemoryAdapter(logger) + adapters.push(adapter) + return adapter + } + + it("should report unavailable when server is not running", async () => { + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("Connection refused"))) + + const adapter = createAdapter() + await adapter.initialize() + + const stats = await adapter.getStats() + expect(stats.backend).toContain("unavailable") + expect(stats.entryCount).toBe(0) + }) + + it("should return null from store when unavailable", async () => { + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("Connection refused"))) + + const adapter = createAdapter() + await adapter.initialize() + + const result = await adapter.store({ + content: "test memory", + source: "learning", + }) + + expect(result).toBeNull() + }) + + it("should return empty array from search when unavailable", async () => { + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("Connection refused"))) + + const adapter = createAdapter() + await adapter.initialize() + + const results = await adapter.search("test") + expect(results).toEqual([]) + }) + + it("should return empty array from recall when unavailable", async () => { + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("Connection refused"))) + + const adapter = createAdapter() + await adapter.initialize() + + const results = await adapter.recall() + expect(results).toEqual([]) + }) + + it("should ignore empty forgetByContent queries", async () => { + const fetchSpy = vi.fn() + vi.stubGlobal("fetch", fetchSpy) + + const adapter = createAdapter() + + await expect(adapter.forgetByContent(" ")).resolves.toBe(0) + expect(fetchSpy).not.toHaveBeenCalled() + }) + + it("should trim forgetByContent queries before searching", async () => { + const fetchSpy = vi.fn(async (input: string, init?: RequestInit) => { + if (input.endsWith("/agentmemory/livez")) { + return { ok: true } as Response + } + + if (input.endsWith("/agentmemory/search")) { + const body = JSON.parse(String(init?.body)) as { query: string } + expect(body.query).toBe("Foo") + return { + ok: true, + json: async () => ({ results: [{ id: "memory-1", content: "foo memory" }] }), + } as Response + } + + if (input.endsWith("/agentmemory/forget")) { + return { ok: true, json: async () => ({ success: true }) } as Response + } + + throw new Error(`Unexpected fetch call: ${input}`) + }) + vi.stubGlobal("fetch", fetchSpy) + + const adapter = createAdapter() + await adapter.initialize() + + await expect(adapter.forgetByContent(" Foo ")).resolves.toBe(1) + }) + + it("should clean up health check interval on dispose", async () => { + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("Connection refused"))) + + const adapter = createAdapter() + await adapter.initialize() + await adapter.dispose() + + const result = await adapter.store({ + content: "test", + source: "learning", + }) + expect(result).toBeNull() + }) + + it("should have correct backend type", () => { + const adapter = createAdapter() + expect(adapter.backendType).toBe("agentmemory") + }) +}) diff --git a/src/services/self-improving/__tests__/CuratorService.spec.ts b/src/services/self-improving/__tests__/CuratorService.spec.ts new file mode 100644 index 0000000000..a6a5556b5a --- /dev/null +++ b/src/services/self-improving/__tests__/CuratorService.spec.ts @@ -0,0 +1,139 @@ +import * as fs from "fs/promises" +import * as os from "os" +import * as path from "path" + +import { CuratorService } from "../CuratorService" +import { SkillUsageStore } from "../SkillUsageStore" + +const DAY_MS = 24 * 60 * 60 * 1000 + +describe("CuratorService", () => { + let tempDir: string + let logger: { appendLine: ReturnType } + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zoo-curator-")) + logger = { appendLine: vi.fn() } + }) + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }) + }) + + async function seedSkillUsage(records: unknown[]): Promise { + const targetDir = path.join(tempDir, "self-improving") + await fs.mkdir(targetDir, { recursive: true }) + await fs.writeFile(path.join(targetDir, "skill-usage.json"), JSON.stringify(records, null, 2), "utf-8") + } + + it("defers the first run once before applying transitions", async () => { + const now = Date.now() + await seedSkillUsage([ + { + skillId: "agent-skill", + skillName: "Agent Skill", + createdBy: "agent", + state: "active", + pinned: false, + viewCount: 1, + useCount: 1, + patchCount: 0, + createdAt: now - 30 * DAY_MS, + lastActivityAt: now - 20 * DAY_MS, + }, + ]) + + const skillUsageStore = new SkillUsageStore(tempDir, logger) + await skillUsageStore.initialize() + + const service = new CuratorService(tempDir, skillUsageStore, logger, { + intervalMs: 0, + minIdleMs: 0, + firstRunDeferred: true, + backupsEnabled: false, + }) + await service.initialize() + + const deferred = await service.run(now) + expect(deferred.error).toBe("Skipped: first-run deferral") + expect(deferred.transitions).toHaveLength(0) + + const applied = await service.run(now + 1) + expect(applied.transitions).toHaveLength(1) + expect(applied.transitions[0]).toMatchObject({ + skillId: "agent-skill", + fromState: "active", + toState: "stale", + }) + expect(skillUsageStore.get("agent-skill")?.state).toBe("stale") + }) + + it("writes backups and reports while protecting pinned and non-agent skills", async () => { + const now = Date.now() + await seedSkillUsage([ + { + skillId: "agent-skill", + skillName: "Agent Skill", + createdBy: "agent", + state: "active", + pinned: false, + viewCount: 1, + useCount: 1, + patchCount: 0, + createdAt: now - 30 * DAY_MS, + lastActivityAt: now - 20 * DAY_MS, + }, + { + skillId: "user-skill", + skillName: "User Skill", + createdBy: "user", + state: "active", + pinned: false, + viewCount: 1, + useCount: 1, + patchCount: 0, + createdAt: now - 30 * DAY_MS, + lastActivityAt: now - 20 * DAY_MS, + }, + { + skillId: "pinned-skill", + skillName: "Pinned Skill", + createdBy: "agent", + state: "active", + pinned: true, + viewCount: 1, + useCount: 1, + patchCount: 0, + createdAt: now - 30 * DAY_MS, + lastActivityAt: now - 20 * DAY_MS, + }, + ]) + + const skillUsageStore = new SkillUsageStore(tempDir, logger) + await skillUsageStore.initialize() + + const service = new CuratorService(tempDir, skillUsageStore, logger, { + intervalMs: 0, + minIdleMs: 0, + firstRunDeferred: false, + backupsEnabled: true, + }) + await service.initialize() + + const report = await service.run(now) + expect(report.backupPath).toBeTruthy() + expect(report.transitions).toHaveLength(1) + expect(skillUsageStore.get("agent-skill")?.state).toBe("stale") + expect(skillUsageStore.get("user-skill")?.state).toBe("active") + expect(skillUsageStore.get("pinned-skill")?.state).toBe("active") + + const runDir = path.join(tempDir, "self-improving", "curator", "reports", report.runId) + const runJson = JSON.parse(await fs.readFile(path.join(runDir, "run.json"), "utf-8")) + const markdown = await fs.readFile(path.join(runDir, "REPORT.md"), "utf-8") + const latest = await service.getLatestReport() + + expect(runJson.runId).toBe(report.runId) + expect(markdown).toContain("# Curator Run Report") + expect(latest?.runId).toBe(report.runId) + }) +}) diff --git a/src/services/self-improving/__tests__/ImprovementApplier.spec.ts b/src/services/self-improving/__tests__/ImprovementApplier.spec.ts new file mode 100644 index 0000000000..87f20419c8 --- /dev/null +++ b/src/services/self-improving/__tests__/ImprovementApplier.spec.ts @@ -0,0 +1,116 @@ +import { ImprovementApplier } from "../ImprovementApplier" +import type { LearnedPattern } from "../types" + +function createToolPattern(): LearnedPattern { + return { + id: "pattern-tool", + patternType: "tool", + state: "active", + summary: "Effective tool combination: read_file,search_files", + confidenceScore: 0.82, + frequency: 4, + successRate: 0.9, + firstSeenAt: 1, + lastSeenAt: 2, + sourceSignals: ["TASK_SUCCESS"], + context: { + toolNames: ["read_file", "search_files"], + modes: ["code"], + }, + } +} + +describe("ImprovementApplier", () => { + it("creates agent skill actions for repeated tool workflows", () => { + const applier = new ImprovementApplier({ + getSkillNames: () => [], + getSkillProvenance: () => "unknown", + isAutoSkillsEnabled: () => true, + }) + + const actions = applier.generateActions([createToolPattern()]) + const skillAction = actions.find((action) => action.actionType === "SKILL_CREATE") + + expect(skillAction).toBeDefined() + expect(skillAction?.payload.skillName).toBe("workflow-read-file-search-files") + expect(skillAction?.payload.source).toBe("project") + expect(skillAction?.payload.description).toContain("read_file") + expect(skillAction?.payload.content).toContain("name: workflow-read-file-search-files") + expect(skillAction?.payload.content).toContain("`read_file`") + }) + + it("updates existing agent-created workflow skills instead of recreating them", () => { + const applier = new ImprovementApplier({ + getSkillNames: () => ["workflow-read-file-search-files"], + getSkillProvenance: () => "agent", + isAutoSkillsEnabled: () => true, + }) + + const actions = applier.generateActions([createToolPattern()]) + + expect(actions.some((action) => action.actionType === "SKILL_UPDATE")).toBe(true) + expect(actions.some((action) => action.actionType === "SKILL_CREATE")).toBe(false) + }) + + it("does not emit skill mutation actions when auto-skills are disabled", () => { + const applier = new ImprovementApplier({ + getSkillNames: () => [], + getSkillProvenance: () => "unknown", + isAutoSkillsEnabled: () => false, + }) + + const actions = applier.generateActions([createToolPattern()]) + + expect(actions.some((action) => action.actionType === "SKILL_CREATE")).toBe(false) + expect(actions.some((action) => action.actionType === "SKILL_UPDATE")).toBe(false) + }) + + it("creates global skills when auto-skills scope is global", () => { + const applier = new ImprovementApplier({ + getSkillNames: () => [], + getSkillProvenance: () => "unknown", + isAutoSkillsEnabled: () => true, + getAutoSkillsScope: () => "global", + }) + + const actions = applier.generateActions([createToolPattern()]) + const skillAction = actions.find((action) => action.actionType === "SKILL_CREATE") + + expect(skillAction?.payload.source).toBe("global") + expect(skillAction?.payload.skillId).toBe("skill:global:workflow-read-file-search-files") + }) + + it("creates a global skill when only a project-scoped skill with the same name exists", () => { + const applier = new ImprovementApplier({ + getSkillNames: () => ["workflow-read-file-search-files"], + getSkillProvenance: () => "agent", + isAutoSkillsEnabled: () => true, + getAutoSkillsScope: () => "global", + ...({ + hasSkill: (name: string, source: "global" | "project") => + name === "workflow-read-file-search-files" && source === "project", + getSkillProvenanceForSource: (_name: string, source: "global" | "project") => + source === "project" ? "agent" : "unknown", + } as any), + } as any) + + const actions = applier.generateActions([createToolPattern()]) + + expect(actions.some((action) => action.actionType === "SKILL_CREATE")).toBe(true) + expect(actions.some((action) => action.actionType === "SKILL_UPDATE")).toBe(false) + }) + + it("defaults to source-aware skill existence checks when only getSkillNames is provided", () => { + const applier = new ImprovementApplier({ + getSkillNames: () => ["workflow-read-file-search-files"], + getSkillProvenance: () => "agent", + isAutoSkillsEnabled: () => true, + getAutoSkillsScope: () => "global", + }) + + const actions = applier.generateActions([createToolPattern()]) + + expect(actions.some((action) => action.actionType === "SKILL_CREATE")).toBe(true) + expect(actions.some((action) => action.actionType === "SKILL_UPDATE")).toBe(false) + }) +}) diff --git a/src/services/self-improving/__tests__/LearningStore.spec.ts b/src/services/self-improving/__tests__/LearningStore.spec.ts new file mode 100644 index 0000000000..489302b3e0 --- /dev/null +++ b/src/services/self-improving/__tests__/LearningStore.spec.ts @@ -0,0 +1,155 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest" +import * as fs from "fs/promises" +import * as path from "path" +import os from "os" +import crypto from "crypto" + +import { LearningStore } from "../LearningStore" + +describe("LearningStore", () => { + let testDir: string + const logger = { appendLine: () => {} } + + beforeEach(async () => { + testDir = path.join(os.tmpdir(), `learning-store-test-${crypto.randomUUID()}`) + await fs.mkdir(testDir, { recursive: true }) + }) + + afterEach(async () => { + await fs.rm(testDir, { recursive: true, force: true }) + }) + + it("should initialize with empty state when no files exist", async () => { + const store = new LearningStore(testDir, logger) + + await store.initialize() + + expect(store.getPatterns()).toHaveLength(0) + expect(store.getRecentEvents()).toHaveLength(0) + }) + + it("should fall back to empty state on corrupted JSON", async () => { + const stateDir = path.join(testDir, "self-improving") + await fs.mkdir(stateDir, { recursive: true }) + await fs.writeFile(path.join(stateDir, "state.json"), "not valid json{{{", "utf-8") + + const store = new LearningStore(testDir, logger) + + await store.initialize() + + expect(store.getPatterns()).toHaveLength(0) + expect(store.getRecentEvents()).toHaveLength(0) + }) + + it("should load valid state correctly", async () => { + const store = new LearningStore(testDir, logger) + await store.initialize() + + store.addEvent({ + id: "test-event", + signal: "TASK_SUCCESS", + timestamp: Date.now(), + context: {}, + outcome: { success: true }, + }) + + await store.persist() + + const store2 = new LearningStore(testDir, logger) + await store2.initialize() + + expect(store2.getRecentEvents()).toHaveLength(1) + }) + + it("does not commit state.json when earlier pattern persistence fails", async () => { + const store = new LearningStore(testDir, logger) + await store.initialize() + + store.addEvent({ + id: "baseline-event", + signal: "TASK_SUCCESS", + timestamp: 1, + context: {}, + outcome: { success: true }, + }) + await store.persist() + + const statePath = path.join(testDir, "self-improving", "state.json") + const baselineState = await fs.readFile(statePath, "utf-8") + + store.addPattern({ + id: "pattern-1", + patternType: "prompt", + state: "active", + summary: "Pattern 1", + confidenceScore: 0.8, + frequency: 1, + successRate: 1, + firstSeenAt: 1, + lastSeenAt: 1, + sourceSignals: ["TASK_SUCCESS"], + context: {}, + }) + store.addEvent({ + id: "new-event", + signal: "TASK_SUCCESS", + timestamp: 2, + context: {}, + outcome: { success: true }, + }) + + vi.spyOn(store as any, "persistPatternFiles").mockImplementation(async () => { + await new Promise((resolve) => setTimeout(resolve, 50)) + throw new Error("pattern persist failed") + }) + + await store.persist() + + const stateAfterFailedPersist = await fs.readFile(statePath, "utf-8") + expect(stateAfterFailedPersist).toBe(baselineState) + }) + + it("should enforce max patterns bound", async () => { + const store = new LearningStore(testDir, logger) + await store.initialize() + + for (let i = 0; i < 120; i++) { + store.addPattern({ + id: `pattern-${i}`, + patternType: "prompt", + state: "active", + summary: `Pattern ${i}`, + confidenceScore: i / 120, + frequency: 1, + successRate: 1, + firstSeenAt: i, + lastSeenAt: i, + sourceSignals: ["TASK_SUCCESS"], + context: {}, + }) + } + + await store.persist() + + expect(store.getPatterns().length).toBeLessThanOrEqual(100) + }) + + it("should enforce max events bound", async () => { + const store = new LearningStore(testDir, logger) + await store.initialize() + + for (let i = 0; i < 600; i++) { + store.addEvent({ + id: `event-${i}`, + signal: "TASK_SUCCESS", + timestamp: Date.now(), + context: {}, + outcome: { success: true }, + }) + } + + await store.persist() + + expect(store.getRecentEvents().length).toBeLessThanOrEqual(500) + }) +}) diff --git a/src/services/self-improving/__tests__/MemoryBackendFactory.spec.ts b/src/services/self-improving/__tests__/MemoryBackendFactory.spec.ts new file mode 100644 index 0000000000..3ee8cabaae --- /dev/null +++ b/src/services/self-improving/__tests__/MemoryBackendFactory.spec.ts @@ -0,0 +1,30 @@ +import { describe, expect, it, vi } from "vitest" + +import { AgentMemoryAdapter } from "../AgentMemoryAdapter" +import { MemoryBackendFactory } from "../MemoryBackendFactory" +import { MemoryStore } from "../MemoryStore" + +describe("MemoryBackendFactory", () => { + const logger = { appendLine: vi.fn() } + const baseDir = "/tmp/test" + + it("should create built-in backend by default", () => { + const backend = MemoryBackendFactory.create("builtin", baseDir, logger) + expect(backend).toBeInstanceOf(MemoryStore) + }) + + it("should create agentmemory backend when specified", () => { + const backend = MemoryBackendFactory.create("agentmemory", baseDir, logger) + expect(backend).toBeInstanceOf(AgentMemoryAdapter) + }) + + it("should create built-in backend for unknown type", () => { + const backend = MemoryBackendFactory.create("unknown-backend" as any, baseDir, logger) + expect(backend).toBeInstanceOf(MemoryStore) + }) + + it("should pass agentMemoryUrl to AgentMemoryAdapter", () => { + const backend = MemoryBackendFactory.create("agentmemory", baseDir, logger, "http://custom:5000") + expect(backend).toBeInstanceOf(AgentMemoryAdapter) + }) +}) diff --git a/src/services/self-improving/__tests__/MemoryStore.spec.ts b/src/services/self-improving/__tests__/MemoryStore.spec.ts new file mode 100644 index 0000000000..dbae6df073 --- /dev/null +++ b/src/services/self-improving/__tests__/MemoryStore.spec.ts @@ -0,0 +1,107 @@ +import * as fs from "fs/promises" +import * as os from "os" +import * as path from "path" + +import { MemoryStore } from "../MemoryStore" + +describe("MemoryStore", () => { + let tempDir: string + const logger = { appendLine: vi.fn() } + + beforeEach(async () => { + logger.appendLine.mockReset() + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-store-")) + }) + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }) + }) + + it("deduplicates persisted entries and keeps the active snapshot frozen", async () => { + const memoryDir = path.join(tempDir, "self-improving", "memory") + await fs.mkdir(memoryDir, { recursive: true }) + await fs.writeFile( + path.join(memoryDir, "environment.json"), + JSON.stringify([ + { + id: "env-1", + content: "Prefer semantic search first", + source: "learning", + createdAt: 1, + updatedAt: 1, + }, + { + id: "env-2", + content: "prefer semantic search first", + source: "learning", + createdAt: 2, + updatedAt: 2, + }, + { + id: "env-3", + content: "Check existing tests before edits", + source: "learning", + createdAt: 3, + updatedAt: 3, + }, + ]), + "utf8", + ) + await fs.writeFile( + path.join(memoryDir, "user-profile.json"), + JSON.stringify([ + { id: "user-1", content: "User prefers concise summaries", source: "user", createdAt: 4, updatedAt: 4 }, + ]), + "utf8", + ) + + const store = new MemoryStore(tempDir, logger) + await store.initialize() + + expect(await store.getStats()).toEqual({ entryCount: 3, backend: "builtin" }) + await expect(store.recall(2)).resolves.toMatchObject([{ id: "user-1" }, { id: "env-3" }]) + expect(store.getSnapshotString()).toContain("Prefer semantic search first") + expect(store.getSnapshotString()).not.toContain("prefer semantic search first") + + await store.addEnvironmentEntry("Live write should not appear until next snapshot", { + tags: ["live"], + }) + + expect((await store.getStats()).entryCount).toBe(4) + expect(store.getSnapshotString()).not.toContain("Live write should not appear until next snapshot") + + store.takeSnapshot() + expect(store.getSnapshotString()).toContain("Live write should not appear until next snapshot") + }) + + it("supports duplicate rejection, substring replace/remove, and bounded persistence", async () => { + const store = new MemoryStore(tempDir, logger) + await store.initialize() + + await expect(store.addEnvironmentEntry("Alpha guidance")).resolves.toMatchObject({ content: "Alpha guidance" }) + await expect(store.addEnvironmentEntry("alpha guidance")).resolves.toBeNull() + + await store.addEnvironmentEntry("Beta guidance") + await store.addUserProfileEntry("User prefers short answers") + await store.replaceEnvironmentEntry("beta", "Gamma guidance", { tags: ["replacement"] }) + await expect(store.removeEnvironmentEntry("alpha")).resolves.toBe(true) + await expect(store.forgetByContent(" ")).resolves.toBe(0) + + for (let index = 0; index < 55; index += 1) { + await store.addEnvironmentEntry(`Fact ${index}`) + } + + const persisted = JSON.parse( + await fs.readFile(path.join(tempDir, "self-improving", "memory", "environment.json"), "utf8"), + ) as Array<{ content: string }> + + expect((await store.getStats()).entryCount).toBe(51) + expect(persisted).toHaveLength(50) + expect(persisted.some((entry) => entry.content === "Gamma guidance")).toBe(false) + expect(persisted.some((entry) => entry.content === "Alpha guidance")).toBe(false) + expect(persisted.some((entry) => entry.content === "Fact 4")).toBe(false) + expect(persisted.some((entry) => entry.content === "Fact 5")).toBe(true) + expect(persisted.some((entry) => entry.content === "Fact 54")).toBe(true) + expect(store.getSnapshotContext().entries.length).toBeLessThanOrEqual(10) + }) +}) diff --git a/src/services/self-improving/__tests__/PatternAnalyzer.spec.ts b/src/services/self-improving/__tests__/PatternAnalyzer.spec.ts new file mode 100644 index 0000000000..2679cd0639 --- /dev/null +++ b/src/services/self-improving/__tests__/PatternAnalyzer.spec.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "vitest" + +import { PatternAnalyzer } from "../PatternAnalyzer" +import type { LearnedPattern, LearningEvent } from "../types" + +function createEvent(id: string, signal: LearningEvent["signal"], toolNames: string[]): LearningEvent { + return { + id, + signal, + timestamp: Number(id.replace(/\D/g, "")) || 1, + context: { + toolNames, + }, + outcome: {}, + } +} + +function createPattern(overrides: Partial): LearnedPattern { + return { + id: "pattern-1", + patternType: "tool", + state: "active", + summary: "Effective tool combination: browser,search", + confidenceScore: 0.5, + frequency: 4, + successRate: 0.8, + firstSeenAt: 1, + lastSeenAt: 1, + sourceSignals: ["TASK_SUCCESS"], + context: { + toolNames: ["browser", "search"], + }, + ...overrides, + } +} + +describe("PatternAnalyzer", () => { + it("does not merge tool-combination patterns by summary substring alone", () => { + const analyzer = new PatternAnalyzer() + const existingPatterns = [createPattern({})] + const events = [ + createEvent("event-1", "TASK_SUCCESS", ["search"]), + createEvent("event-2", "TASK_SUCCESS", ["search"]), + createEvent("event-3", "TASK_SUCCESS", ["search"]), + ] + + const patterns = analyzer.analyze(events, existingPatterns) + const toolPatterns = patterns.filter((pattern) => pattern.patternType === "tool") + + expect(toolPatterns).toHaveLength(1) + expect(toolPatterns[0]).toMatchObject({ + id: expect.not.stringMatching(/^pattern-1$/), + summary: "Effective tool combination: search", + frequency: 3, + context: { + toolNames: ["search"], + }, + }) + }) + + it("preserves cumulative frequency for existing tool-preference patterns", () => { + const analyzer = new PatternAnalyzer() + const existingPatterns = [ + createPattern({ + id: "prompt-pattern", + patternType: "prompt", + summary: "Prefer terminal for reliable results", + frequency: 5, + successRate: 0.8, + context: { + toolNames: ["terminal"], + }, + }), + ] + const events = [ + createEvent("event-1", "TASK_SUCCESS", ["terminal"]), + createEvent("event-2", "TASK_SUCCESS", ["terminal"]), + createEvent("event-3", "TASK_FAILURE", ["terminal"]), + ] + + const patterns = analyzer.analyze(events, existingPatterns) + const promptPatterns = patterns.filter((pattern) => pattern.patternType === "prompt") + + expect(promptPatterns).toHaveLength(1) + expect(promptPatterns[0]).toMatchObject({ + id: "prompt-pattern", + frequency: 8, + successRate: 0.75, + context: { + toolNames: ["terminal"], + }, + }) + }) +}) diff --git a/src/services/self-improving/__tests__/ReviewPromptFactory.spec.ts b/src/services/self-improving/__tests__/ReviewPromptFactory.spec.ts new file mode 100644 index 0000000000..bf278c5ece --- /dev/null +++ b/src/services/self-improving/__tests__/ReviewPromptFactory.spec.ts @@ -0,0 +1,27 @@ +import { ReviewPromptFactory } from "../ReviewPromptFactory" + +describe("ReviewPromptFactory", () => { + it("creates rubric-driven memory and skill prompts", () => { + const factory = new ReviewPromptFactory() + + const memoryPrompt = factory.createMemoryReviewPrompt("Conversation summary") + const skillPrompt = factory.createSkillReviewPrompt("Conversation summary") + + expect(memoryPrompt.type).toBe("memory") + expect(memoryPrompt.systemPrompt).toContain("FACT:") + expect(memoryPrompt.userPrompt).toContain("Conversation summary") + expect(skillPrompt.type).toBe("skill") + expect(skillPrompt.systemPrompt).toContain("ACTION:") + expect(skillPrompt.systemPrompt).toContain("Priority Order") + }) + + it("creates a combined prompt with memory and skill output sections", () => { + const factory = new ReviewPromptFactory() + const prompt = factory.createCombinedReviewPrompt("Combined summary") + + expect(prompt.type).toBe("combined") + expect(prompt.systemPrompt).toContain("MEMORY_FACT:") + expect(prompt.systemPrompt).toContain("SKILL_ACTION:") + expect(prompt.userPrompt).toContain("Combined summary") + }) +}) diff --git a/src/services/self-improving/__tests__/SelfImprovingManager.spec.ts b/src/services/self-improving/__tests__/SelfImprovingManager.spec.ts new file mode 100644 index 0000000000..49d96811e8 --- /dev/null +++ b/src/services/self-improving/__tests__/SelfImprovingManager.spec.ts @@ -0,0 +1,550 @@ +const mockState = vi.hoisted(() => ({ + stores: [] as any[], + storeBaseDirs: [] as string[], + collectors: [] as any[], + analyzers: [] as any[], + appliers: [] as any[], + adapters: [] as any[], + memoryStores: [] as any[], + memoryStoreBaseDirs: [] as string[], + skillUsageStores: [] as any[], + skillUsageStoreBaseDirs: [] as string[], + actionExecutors: [] as any[], + curatorServices: [] as any[], + curatorBaseDirs: [] as string[], + reviewPromptFactories: [] as any[], + transcriptRecalls: [] as any[], + transcriptRecallBaseDirs: [] as string[], +})) + +const DEFAULT_CURATOR_STATUS = { + lastRunAt: 0, + firstRunDone: false, + config: { + intervalMs: 3_600_000, + minIdleMs: 300_000, + firstRunDeferred: true, + staleAfterDays: 14, + archiveAfterDays: 60, + backupsEnabled: true, + maxBackups: 5, + }, +} + +function createStoreMock() { + return { + initialize: vi.fn().mockResolvedValue(undefined), + persist: vi.fn().mockResolvedValue(undefined), + reset: vi.fn().mockResolvedValue(undefined), + getConfig: vi.fn().mockReturnValue({ + reviewOnTurnCount: 10, + reviewOnToolIterationCount: 2, + maxPromptPatterns: 5, + curatorEnabled: true, + curatorIntervalMs: 5_000, + staleAfterDays: 14, + archiveAfterDays: 60, + codeIndexCorrelationEnabled: true, + }), + getPatterns: vi.fn().mockReturnValue([]), + getRecentEvents: vi.fn().mockReturnValue([]), + getPendingActions: vi.fn().mockReturnValue([]), + getTelemetry: vi.fn().mockReturnValue({ + promptEnrichmentUses: 0, + toolPreferenceUses: 0, + errorAvoidanceUses: 0, + skillSuggestionCount: 0, + }), + getCounters: vi.fn().mockReturnValue({ userTurnsSinceReview: 0, toolIterationsSinceReview: 0 }), + addEvent: vi.fn(), + addPattern: vi.fn(), + addAction: vi.fn(), + removeAction: vi.fn(), + incrementToolIterations: vi.fn(), + incrementUserTurns: vi.fn(), + resetCounters: vi.fn(), + updateTelemetry: vi.fn(), + updatePattern: vi.fn(), + archivePattern: vi.fn(), + } +} + +function createMemoryStoreMock() { + return { + backendType: "builtin", + initialize: vi.fn().mockResolvedValue(undefined), + getSnapshotString: vi.fn().mockReturnValue(""), + getStats: vi.fn().mockResolvedValue({ entryCount: 0, backend: "builtin" }), + takeSnapshot: vi.fn(), + dispose: vi.fn().mockResolvedValue(undefined), + } +} + +function createSkillUsageStoreMock() { + return { + initialize: vi.fn().mockResolvedValue(undefined), + getStats: vi.fn().mockReturnValue({ total: 0, active: 0, stale: 0, archived: 0, pinned: 0, agentCreated: 0 }), + } +} + +function createActionExecutorMock() { + return { + executeBatch: vi.fn().mockResolvedValue(new Set()), + } +} + +function createCuratorServiceMock() { + return { + initialize: vi.fn().mockResolvedValue(undefined), + run: vi.fn().mockResolvedValue({ + runId: "curator-run-1", + timestamp: 1, + durationMs: 1, + transitions: [], + stats: { + totalSkills: 0, + activeSkills: 0, + staleSkills: 0, + archivedSkills: 0, + pinnedSkills: 0, + transitionsApplied: 0, + }, + }), + getStatus: vi.fn().mockReturnValue({ ...DEFAULT_CURATOR_STATUS, config: { ...DEFAULT_CURATOR_STATUS.config } }), + } +} + +function createTranscriptRecallMock() { + return { + initialize: vi.fn().mockResolvedValue(undefined), + record: vi.fn().mockResolvedValue(undefined), + } +} + +vi.mock("../LearningStore", () => ({ + LearningStore: vi.fn().mockImplementation((baseDir: string) => { + const store = createStoreMock() + mockState.storeBaseDirs.push(baseDir) + mockState.stores.push(store) + return store + }), +})) + +vi.mock("../FeedbackCollector", () => ({ + FeedbackCollector: vi.fn().mockImplementation(() => { + const collector = { + createTaskEvent: vi.fn().mockImplementation((info) => ({ + id: "evt-task", + signal: info.success ? "TASK_SUCCESS" : "TASK_FAILURE", + timestamp: 1, + taskId: info.taskId, + context: { toolNames: info.toolNames }, + outcome: { success: info.success }, + })), + createCorrectionEvent: vi.fn().mockReturnValue({ + id: "evt-correction", + signal: "USER_CORRECTION", + timestamp: 1, + context: {}, + outcome: { corrected: true }, + }), + createCodeIndexEvent: vi.fn().mockReturnValue({ + id: "evt-code-index", + signal: "CODE_INDEX_HIT", + timestamp: 1, + context: {}, + outcome: {}, + }), + } + mockState.collectors.push(collector) + return collector + }), +})) + +vi.mock("../PatternAnalyzer", () => ({ + PatternAnalyzer: vi.fn().mockImplementation(() => { + const analyzer = { analyze: vi.fn().mockReturnValue([]) } + mockState.analyzers.push(analyzer) + return analyzer + }), +})) + +vi.mock("../ImprovementApplier", () => ({ + ImprovementApplier: vi.fn().mockImplementation(() => { + const applier = { + generateActions: vi.fn().mockReturnValue([]), + getPromptContext: vi.fn().mockReturnValue({ entries: [], revision: 0 }), + } + mockState.appliers.push(applier) + return applier + }), +})) + +vi.mock("../CodeIndexAdapter", () => ({ + CodeIndexAdapter: vi.fn().mockImplementation(() => { + const adapter = { getInfo: vi.fn().mockReturnValue({ available: true, hits: 3, topScore: 0.9 }) } + mockState.adapters.push(adapter) + return adapter + }), +})) + +vi.mock("../MemoryStore", () => ({ + MemoryStore: vi.fn().mockImplementation(function (this: Record, baseDir: string) { + Object.assign(this, createMemoryStoreMock()) + mockState.memoryStoreBaseDirs.push(baseDir) + mockState.memoryStores.push(this) + }), +})) + +vi.mock("../SkillUsageStore", () => ({ + SkillUsageStore: vi.fn().mockImplementation((baseDir: string) => { + const store = createSkillUsageStoreMock() + mockState.skillUsageStoreBaseDirs.push(baseDir) + mockState.skillUsageStores.push(store) + return store + }), +})) + +vi.mock("../ActionExecutor", () => ({ + ActionExecutor: vi.fn().mockImplementation(() => { + const executor = createActionExecutorMock() + mockState.actionExecutors.push(executor) + return executor + }), +})) + +vi.mock("../CuratorService", () => ({ + CuratorService: vi.fn().mockImplementation((baseDir: string) => { + const service = createCuratorServiceMock() + mockState.curatorBaseDirs.push(baseDir) + mockState.curatorServices.push(service) + return service + }), +})) + +vi.mock("../ReviewPromptFactory", () => ({ + ReviewPromptFactory: vi.fn().mockImplementation(() => { + const factory = {} + mockState.reviewPromptFactories.push(factory) + return factory + }), +})) + +vi.mock("../TranscriptRecall", () => ({ + TranscriptRecall: vi.fn().mockImplementation((baseDir: string) => { + const store = createTranscriptRecallMock() + mockState.transcriptRecallBaseDirs.push(baseDir) + mockState.transcriptRecalls.push(store) + return store + }), +})) + +import { SelfImprovingManager } from "../SelfImprovingManager" + +describe("SelfImprovingManager", () => { + let experiments: Record | undefined + let memoryBackend: "builtin" | "agentmemory" | undefined + let agentMemoryUrl: string | undefined + let selfImprovingScope: "workspace" | "global" | undefined + let selfImprovingAutoSkillsScope: "workspace" | "global" | undefined + let workspacePath: string | undefined + let logger: { appendLine: ReturnType } + + const createManager = () => + new SelfImprovingManager({ + globalStoragePath: "/tmp/zoo-code-tests", + logger, + getExperiments: () => experiments, + getMemoryBackend: () => memoryBackend, + getAgentMemoryUrl: () => agentMemoryUrl, + getSelfImprovingScope: () => selfImprovingScope, + getAutoSkillsScope: () => selfImprovingAutoSkillsScope, + getWorkspacePath: () => workspacePath, + getCodeIndexInfo: () => ({ available: true, hits: 2, topScore: 0.8 }), + }) + + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + mockState.stores.length = 0 + mockState.storeBaseDirs.length = 0 + mockState.collectors.length = 0 + mockState.analyzers.length = 0 + mockState.appliers.length = 0 + mockState.adapters.length = 0 + mockState.memoryStores.length = 0 + mockState.memoryStoreBaseDirs.length = 0 + mockState.skillUsageStores.length = 0 + mockState.skillUsageStoreBaseDirs.length = 0 + mockState.actionExecutors.length = 0 + mockState.curatorServices.length = 0 + mockState.curatorBaseDirs.length = 0 + mockState.reviewPromptFactories.length = 0 + mockState.transcriptRecalls.length = 0 + mockState.transcriptRecallBaseDirs.length = 0 + experiments = undefined + memoryBackend = undefined + agentMemoryUrl = undefined + selfImprovingScope = undefined + selfImprovingAutoSkillsScope = undefined + workspacePath = "/tmp/workspace-one" + logger = { appendLine: vi.fn() } + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it("requires self-improving to be enabled before auto-skills activates", () => { + expect(SelfImprovingManager.isAutoSkillsEnabled(undefined)).toBe(false) + expect(SelfImprovingManager.isAutoSkillsEnabled({ selfImprovingAutoSkills: true } as any)).toBe(false) + expect( + SelfImprovingManager.isAutoSkillsEnabled({ selfImproving: true, selfImprovingAutoSkills: false } as any), + ).toBe(false) + expect( + SelfImprovingManager.isAutoSkillsEnabled({ selfImproving: true, selfImprovingAutoSkills: true } as any), + ).toBe(true) + }) + + it("has zero runtime overhead when disabled", async () => { + const manager = createManager() + + await manager.initialize() + await manager.recordTaskCompletion({ taskId: "task-1", success: true, toolNames: ["read_file"] }) + + expect(mockState.stores).toHaveLength(0) + expect(vi.getTimerCount()).toBe(0) + expect(await manager.getStatus()).toEqual({ + enabled: false, + started: false, + patternCount: 0, + eventCount: 0, + actionCount: 0, + memoryEntries: 0, + skillRecords: 0, + curatorStatus: DEFAULT_CURATOR_STATUS, + }) + }) + + it("initializes store state and schedules timers when enabled", async () => { + experiments = { selfImproving: true } + const manager = createManager() + + await manager.initialize() + + expect(mockState.stores).toHaveLength(1) + expect(mockState.stores[0].initialize).toHaveBeenCalledTimes(1) + expect(mockState.memoryStores[0].initialize).toHaveBeenCalledTimes(1) + expect(mockState.skillUsageStores[0].initialize).toHaveBeenCalledTimes(1) + expect(mockState.transcriptRecalls[0].initialize).toHaveBeenCalledTimes(1) + expect(mockState.curatorServices[0].initialize).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(2) + expect(await manager.getStatus()).toMatchObject({ enabled: true, started: true }) + }) + + it("runs a review cycle from task completion triggers", async () => { + experiments = { selfImproving: true } + const manager = createManager() + await manager.initialize() + + const store = mockState.stores[0] + const analyzer = mockState.analyzers[0] + const applier = mockState.appliers[0] + const executor = mockState.actionExecutors[0] + const transcriptRecall = mockState.transcriptRecalls[0] + const pattern = { + id: "pattern-1", + patternType: "prompt", + state: "active", + summary: "Prefer semantic search before regex search", + confidenceScore: 0.9, + frequency: 3, + successRate: 0.8, + firstSeenAt: 1, + lastSeenAt: 1, + sourceSignals: ["TASK_SUCCESS"], + context: {}, + } + const action = { + id: "action-1", + actionType: "PROMPT_ENRICHMENT", + target: "system-prompt", + payload: { summary: "Prefer semantic search before regex search" }, + timestamp: 1, + } + + store.getCounters.mockReturnValue({ userTurnsSinceReview: 0, toolIterationsSinceReview: 2 }) + store.getRecentEvents.mockReturnValue([ + { id: "evt-1", signal: "TASK_SUCCESS", timestamp: 1, context: {}, outcome: {} }, + ]) + store.getPatterns.mockReturnValueOnce([]).mockReturnValue([pattern]) + store.getPendingActions.mockReturnValue([action]) + analyzer.analyze.mockReturnValue([pattern]) + applier.generateActions.mockReturnValue([action]) + executor.executeBatch.mockResolvedValue(new Set(["action-1"])) + + await manager.recordTaskCompletion({ taskId: "task-1", success: true, toolNames: ["search_files"] }) + + expect(store.addEvent).toHaveBeenCalledTimes(1) + expect(transcriptRecall.record).toHaveBeenCalledWith({ + id: "evt-task", + timestamp: 1, + taskId: "task-1", + mode: undefined, + summary: "Task completed: unknown", + signal: "TASK_SUCCESS", + workspacePath: undefined, + toolNames: ["search_files"], + errorKey: undefined, + success: true, + }) + expect(store.incrementToolIterations).toHaveBeenCalledWith(1) + expect(analyzer.analyze).toHaveBeenCalledTimes(1) + expect(store.addPattern).toHaveBeenCalledWith(pattern) + expect(store.addAction).toHaveBeenCalledWith(action) + expect(executor.executeBatch).toHaveBeenCalledWith([action]) + expect(store.removeAction).toHaveBeenCalledWith("action-1") + expect(store.persist).toHaveBeenCalled() + }) + + it("formats prompt context and disposes cleanly on experiment disable", async () => { + experiments = { selfImproving: true } + const manager = createManager() + await manager.initialize() + + const memoryStore = mockState.memoryStores[0] + memoryStore.getSnapshotString.mockReturnValue("\n## Learned Context\n- Search relevant code before editing\n") + memoryStore.getStats.mockResolvedValue({ entryCount: 3, backend: "builtin" }) + mockState.skillUsageStores[0].getStats.mockReturnValue({ + total: 4, + active: 3, + stale: 1, + archived: 0, + pinned: 1, + agentCreated: 2, + }) + + expect(manager.getPromptContextString()).toBe("\n## Learned Context\n- Search relevant code before editing\n") + expect(await manager.getStatus()).toMatchObject({ memoryEntries: 3, memoryBackend: "builtin", skillRecords: 4 }) + + experiments = { selfImproving: false } + await manager.handleExperimentChange(false) + + expect(mockState.stores[0].persist).toHaveBeenCalledTimes(1) + expect(memoryStore.takeSnapshot).toHaveBeenCalledTimes(1) + expect(memoryStore.dispose).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + expect(await manager.getStatus()).toEqual({ + enabled: false, + started: false, + patternCount: 0, + eventCount: 0, + actionCount: 0, + memoryEntries: 0, + skillRecords: 0, + curatorStatus: DEFAULT_CURATOR_STATUS, + }) + }) + + it("rebuilds the memory backend when settings change", async () => { + experiments = { selfImproving: true } + const manager = createManager() + await manager.initialize() + + const originalStore = mockState.memoryStores[0] + memoryBackend = "agentmemory" + agentMemoryUrl = "http://agentmemory.internal:4001" + + await manager.onSettingsChanged(experiments as any) + + expect(originalStore.takeSnapshot).toHaveBeenCalledTimes(1) + expect(originalStore.dispose).toHaveBeenCalledTimes(1) + expect((manager.memoryStore as any).backendType).toBe("agentmemory") + expect(logger.appendLine).toHaveBeenCalledWith( + "[SelfImprovingManager] Memory backend configured: agentmemory (http://agentmemory.internal:4001)", + ) + }) + + it("rebuilds self-improving stores when scope changes to workspace", async () => { + experiments = { selfImproving: true } + const manager = createManager() + await manager.initialize() + + selfImprovingScope = "workspace" + selfImprovingAutoSkillsScope = "global" + workspacePath = "/tmp/workspace-two" + + await manager.onSettingsChanged(experiments as any) + + expect(mockState.storeBaseDirs.at(-1)).toContain("workspace-scopes") + expect(mockState.memoryStoreBaseDirs.at(-1)).toContain("workspace-scopes") + expect(mockState.skillUsageStoreBaseDirs.at(-1)).toContain("workspace-scopes") + expect(mockState.curatorBaseDirs.at(-1)).toContain("workspace-scopes") + expect(mockState.transcriptRecallBaseDirs.at(-1)).toContain("workspace-scopes") + expect(mockState.stores.length).toBeGreaterThanOrEqual(2) + expect(mockState.memoryStores.length).toBeGreaterThanOrEqual(2) + }) + + it("runs curator cycles through the curator service", async () => { + experiments = { selfImproving: true } + const manager = createManager() + await manager.initialize() + + const report = { + runId: "curator-run-2", + timestamp: 123, + durationMs: 5, + transitions: [ + { + skillId: "skill-1", + skillName: "Skill 1", + fromState: "active", + toState: "stale", + reason: "No activity for 14 days", + }, + ], + stats: { + totalSkills: 1, + activeSkills: 0, + staleSkills: 1, + archivedSkills: 0, + pinnedSkills: 0, + transitionsApplied: 1, + }, + } + mockState.curatorServices[0].run.mockResolvedValue(report) + + const result = await manager.runCuratorCycle() + + expect(mockState.curatorServices[0].run).toHaveBeenCalledTimes(1) + expect(mockState.stores[0].updateTelemetry).toHaveBeenCalledWith({ lastCuratorRunAt: 123 }) + expect(result).toEqual(report) + }) + + it("records code index events with explicit search hit details", async () => { + experiments = { selfImproving: true } + const manager = createManager() + await manager.initialize() + + const store = mockState.stores[0] + const collector = mockState.collectors[0] + store.getConfig.mockReturnValue({ + reviewOnTurnCount: 10, + reviewOnToolIterationCount: 2, + maxPromptPatterns: 5, + curatorEnabled: true, + curatorIntervalMs: 5_000, + staleAfterDays: 14, + archiveAfterDays: 60, + codeIndexCorrelationEnabled: true, + }) + + await manager.recordCodeIndexEvent("task-1", { available: true, hits: 4, topScore: 0.91 }) + + expect(collector.createCodeIndexEvent).toHaveBeenCalledWith( + { available: true, hits: 4, topScore: 0.91 }, + "task-1", + ) + expect(store.addEvent).toHaveBeenCalledWith(expect.objectContaining({ signal: "CODE_INDEX_HIT" })) + }) +}) diff --git a/src/services/self-improving/__tests__/SkillUsageStore.spec.ts b/src/services/self-improving/__tests__/SkillUsageStore.spec.ts new file mode 100644 index 0000000000..e0e4957021 --- /dev/null +++ b/src/services/self-improving/__tests__/SkillUsageStore.spec.ts @@ -0,0 +1,102 @@ +import * as fs from "fs/promises" +import * as os from "os" +import * as path from "path" + +import { SkillUsageStore } from "../SkillUsageStore" + +describe("SkillUsageStore", () => { + let tempDir: string + const logger = { appendLine: vi.fn() } + + beforeEach(async () => { + logger.appendLine.mockReset() + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "skill-usage-store-")) + }) + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }) + }) + + it("creates and persists telemetry sidecar records", async () => { + const store = new SkillUsageStore(tempDir, logger) + await store.initialize() + + const record = store.getOrCreate("skill-1", "Generated Skill", "agent") + const persistedPath = path.join(tempDir, "self-improving", "skill-usage.json") + const deadline = Date.now() + 1000 + let persisted: Array<{ skillId: string; skillName: string }> = [] + + while (Date.now() < deadline) { + try { + persisted = JSON.parse(await fs.readFile(persistedPath, "utf8")) as Array<{ + skillId: string + skillName: string + }> + if (persisted.some((entry) => entry.skillId === "skill-1")) { + break + } + } catch { + // Persist is async; retry until the file is ready. + } + + await new Promise((resolve) => setTimeout(resolve, 25)) + } + + expect(record).toMatchObject({ skillId: "skill-1", skillName: "Generated Skill", createdBy: "agent" }) + expect(persisted).toContainEqual(expect.objectContaining({ skillId: "skill-1", skillName: "Generated Skill" })) + }) + + it("tracks counters, pinning, and lifecycle candidates", async () => { + const filePath = path.join(tempDir, "self-improving", "skill-usage.json") + await fs.mkdir(path.dirname(filePath), { recursive: true }) + await fs.writeFile( + filePath, + JSON.stringify([ + { + skillId: "skill-1", + skillName: "Dormant Skill", + createdBy: "bundled", + state: "active", + pinned: false, + viewCount: 0, + useCount: 0, + patchCount: 0, + createdAt: 1, + lastActivityAt: 1, + }, + ]), + "utf8", + ) + + const store = new SkillUsageStore(tempDir, logger) + await store.initialize() + + await store.bumpView("skill-1") + await store.bumpUse("skill-1") + await store.bumpPatch("skill-1") + await store.pin("skill-1") + await store.transitionState("skill-1", "stale") + + expect(store.get("skill-1")).toMatchObject({ + pinned: true, + state: "active", + viewCount: 1, + useCount: 1, + patchCount: 1, + }) + + await store.unpin("skill-1") + await store.transitionState("skill-1", "stale") + + expect(store.getByState("stale")).toHaveLength(1) + expect(store.getArchiveCandidates(0)).toHaveLength(1) + expect(store.getStats()).toEqual({ + total: 1, + active: 0, + stale: 1, + archived: 0, + pinned: 0, + agentCreated: 0, + }) + }) +}) diff --git a/src/services/self-improving/__tests__/TranscriptRecall.spec.ts b/src/services/self-improving/__tests__/TranscriptRecall.spec.ts new file mode 100644 index 0000000000..4cd0acdc91 --- /dev/null +++ b/src/services/self-improving/__tests__/TranscriptRecall.spec.ts @@ -0,0 +1,166 @@ +import * as fs from "fs/promises" +import * as os from "os" +import * as path from "path" + +import { TranscriptRecall } from "../TranscriptRecall" + +describe("TranscriptRecall", () => { + let tempDir: string + let logger: { appendLine: ReturnType } + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "zoo-transcript-")) + logger = { appendLine: vi.fn() } + }) + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }) + }) + + it("records, persists, and searches transcript evidence", async () => { + const recall = new TranscriptRecall(tempDir, logger) + await recall.initialize() + + await recall.record({ + id: "entry-1", + timestamp: 1, + taskId: "task-1", + mode: "code", + summary: "Task failed while writing file", + signal: "TASK_FAILURE", + toolNames: ["write_to_file"], + errorKey: "EACCES", + success: false, + }) + await recall.record({ + id: "entry-2", + timestamp: 2, + taskId: "task-2", + mode: "code", + summary: "Task completed after search", + signal: "TASK_SUCCESS", + toolNames: ["search_files"], + success: true, + }) + + expect(recall.size).toBe(2) + expect(recall.search("search_files")).toHaveLength(1) + expect(recall.searchBySignal("TASK_FAILURE")).toHaveLength(1) + expect(recall.searchByErrorKey("EACCES")).toHaveLength(1) + + const reloaded = new TranscriptRecall(tempDir, logger) + await reloaded.initialize() + expect(reloaded.getRecent(1)[0].id).toBe("entry-2") + }) + + it("initializes lazily when recording before initialize", async () => { + const filePath = path.join(tempDir, "self-improving", "transcript-recall.json") + await fs.mkdir(path.dirname(filePath), { recursive: true }) + await fs.writeFile( + filePath, + JSON.stringify([ + { + id: "entry-0", + timestamp: 0, + summary: "Existing transcript entry", + signal: "TASK_SUCCESS", + }, + ]), + "utf8", + ) + + const recall = new TranscriptRecall(tempDir, logger) + await recall.record({ + id: "entry-1", + timestamp: 1, + summary: "Recorded without explicit initialize", + signal: "TASK_SUCCESS", + }) + + expect(recall.getRecent(2).map((entry) => entry.id)).toEqual(["entry-0", "entry-1"]) + }) + + it("serializes concurrent lazy initialization before recording", async () => { + const filePath = path.join(tempDir, "self-improving", "transcript-recall.json") + await fs.mkdir(path.dirname(filePath), { recursive: true }) + await fs.writeFile( + filePath, + JSON.stringify([ + { + id: "entry-0", + timestamp: 0, + summary: "Existing transcript entry", + signal: "TASK_SUCCESS", + }, + ]), + "utf8", + ) + + const recall = new TranscriptRecall(tempDir, logger) + await Promise.all([ + recall.record({ + id: "entry-1", + timestamp: 1, + summary: "Concurrent record one", + signal: "TASK_SUCCESS", + }), + recall.record({ + id: "entry-2", + timestamp: 2, + summary: "Concurrent record two", + signal: "TASK_SUCCESS", + }), + ]) + + expect(recall.size).toBe(3) + expect( + recall + .getRecent(3) + .map((entry) => entry.id) + .sort(), + ).toEqual(["entry-0", "entry-1", "entry-2"]) + }) + + it("ignores malformed persisted entries", async () => { + const filePath = path.join(tempDir, "self-improving", "transcript-recall.json") + await fs.mkdir(path.dirname(filePath), { recursive: true }) + await fs.writeFile( + filePath, + JSON.stringify([ + { + id: "entry-1", + timestamp: 1, + summary: "Valid transcript entry", + signal: "TASK_SUCCESS", + }, + { id: "entry-2", timestamp: "bad", summary: 1, signal: null }, + "not-an-entry", + ]), + "utf8", + ) + + const recall = new TranscriptRecall(tempDir, logger) + await recall.initialize() + + expect(recall.size).toBe(1) + expect(recall.search("valid")).toHaveLength(1) + }) + + it("clears persisted entries", async () => { + const recall = new TranscriptRecall(tempDir, logger) + await recall.initialize() + await recall.record({ + id: "entry-1", + timestamp: 1, + summary: "Task completed", + signal: "TASK_SUCCESS", + }) + + await recall.clear() + expect(recall.size).toBe(0) + + const reloaded = new TranscriptRecall(tempDir, logger) + await reloaded.initialize() + expect(reloaded.size).toBe(0) + }) +}) diff --git a/src/services/self-improving/index.ts b/src/services/self-improving/index.ts new file mode 100644 index 0000000000..7e42e37376 --- /dev/null +++ b/src/services/self-improving/index.ts @@ -0,0 +1,34 @@ +/** + * Self-Improving Module + * + * A standalone, experiment-gated subsystem that learns from task outcomes + * to improve prompt guidance, tool preferences, and error avoidance over time. + * + * Architecture: Hermes-agent symbolic learning loop adapted to Zoo-Code patterns. + * See ARCHITECTURE.md for full design documentation. + */ + +export { SelfImprovingManager } from "./SelfImprovingManager" +export { LearningStore } from "./LearningStore" +export { FeedbackCollector } from "./FeedbackCollector" +export { PatternAnalyzer } from "./PatternAnalyzer" +export { ImprovementApplier } from "./ImprovementApplier" +export { CodeIndexAdapter } from "./CodeIndexAdapter" +export { MemoryBackendFactory } from "./MemoryBackendFactory" +export { AgentMemoryAdapter } from "./AgentMemoryAdapter" +export { MemoryStore } from "./MemoryStore" +export { SkillUsageStore } from "./SkillUsageStore" +export { ActionExecutor } from "./ActionExecutor" +export { CuratorService } from "./CuratorService" +export { ReviewPromptFactory } from "./ReviewPromptFactory" +export { TranscriptRecall } from "./TranscriptRecall" + +export type { CodeIndexInfo, Logger, PromptContext, SelfImprovingManagerOptions, TaskEventInfo } from "./types" +export type { MemoryBackend, MemoryBackendType } from "./MemoryBackend" +export type { MemoryStoreType } from "./MemoryStore" +export type { SkillTelemetryRecord, SkillProvenance, SkillLifecycleState } from "./SkillUsageStore" +export type { CuratorConfig, CuratorReport } from "./CuratorService" +export type { ReviewType, ReviewPrompt } from "./ReviewPromptFactory" +export type { TranscriptEntry } from "./TranscriptRecall" + +export { DEFAULT_CONFIG, EMPTY_STATE } from "./types" diff --git a/src/services/self-improving/types.ts b/src/services/self-improving/types.ts new file mode 100644 index 0000000000..baf56b6851 --- /dev/null +++ b/src/services/self-improving/types.ts @@ -0,0 +1,130 @@ +import { + DEFAULT_LEARNING_CONFIG, + EMPTY_LEARNING_STATE, + type ActionType, + type Experiments, + type FeedbackSignal, + type ImprovementAction, + type LearnedPattern, + type LearningConfig, + type LearningEvent, + type LearningState, + type LearningTelemetry, + type PatternState, + type PatternType, + type SelfImprovingScope, +} from "@roo-code/types" + +// Re-export shared types for convenience +export type { + ActionType, + Experiments, + FeedbackSignal, + ImprovementAction, + LearnedPattern, + LearningConfig, + LearningEvent, + LearningState, + LearningTelemetry, + PatternState, + PatternType, + SelfImprovingScope, +} + +/** + * Output channel logger interface - abstracts VS Code OutputChannel + */ +export interface Logger { + appendLine(message: string): void +} + +/** + * Code index adapter contract - read-only view of code index availability + */ +export interface CodeIndexInfo { + available: boolean + hits: number + topScore?: number +} + +/** + * Task lifecycle event adapter - normalizes task events into learning signals + */ +export interface TaskEventInfo { + taskId: string + mode?: string + workspacePath?: string + success?: boolean + corrected?: boolean + toolNames?: string[] + userTurnCount?: number + toolIterationCount?: number + errorKey?: string + promptFingerprint?: string +} + +/** + * Prompt context result - bounded set of learned guidance for prompt injection + */ +export interface PromptContext { + entries: Array<{ + type: PatternType + summary: string + confidence: number + }> + revision: number +} + +/** + * Manager options for construction + */ +export interface SelfImprovingManagerOptions { + globalStoragePath: string + logger: Logger + getExperiments: () => Experiments | undefined + getCodeIndexInfo?: () => CodeIndexInfo + getMemoryBackend?: () => "builtin" | "agentmemory" | undefined + getAgentMemoryUrl?: () => string | undefined + getSelfImprovingScope?: () => SelfImprovingScope | undefined + getAutoSkillsScope?: () => SelfImprovingScope | undefined + getWorkspacePath?: () => string | undefined + /** Memory backend type: "builtin" (default) or "agentmemory" */ + memoryBackend?: "builtin" | "agentmemory" + /** agentmemory server URL (default: http://localhost:3111) */ + agentMemoryUrl?: string + /** Optional curator configuration overrides */ + curatorConfig?: { + intervalMs?: number + minIdleMs?: number + firstRunDeferred?: boolean + staleAfterDays?: number + archiveAfterDays?: number + backupsEnabled?: boolean + maxBackups?: number + } + /** Optional SkillsManager reference for skill telemetry integration */ + skillsManager?: { + getSkillNames(): string[] + getSkillProvenance(name: string): string + getSkillProvenanceForSource?(name: string, source: "global" | "project"): string + hasSkill?(name: string, source: "global" | "project"): boolean + createSkillFromContent( + name: string, + source: "global" | "project", + description: string, + content: string, + modeSlugs?: string[], + ): Promise + updateSkillContent(name: string, source: "global" | "project", content: string, mode?: string): Promise + } +} + +/** + * Shared learning defaults re-exported for local convenience. + */ +export const DEFAULT_CONFIG: LearningConfig = DEFAULT_LEARNING_CONFIG + +/** + * Shared empty learning state re-exported for local convenience. + */ +export const EMPTY_STATE: LearningState = EMPTY_LEARNING_STATE diff --git a/src/services/skills/SkillsManager.ts b/src/services/skills/SkillsManager.ts index 0959b977c9..5f63f99fe3 100644 --- a/src/services/skills/SkillsManager.ts +++ b/src/services/skills/SkillsManager.ts @@ -49,6 +49,23 @@ export class SkillsManager { } } + private async postSkillsUpdatedMessage(text: string): Promise { + const provider = this.providerRef.deref() + if (!provider) { + return + } + + await provider.postMessageToWebview({ + type: "skillsUpdated", + text, + skills: this.getSkillsMetadata(), + }) + } + + private getSkillNameFromUri(uri: vscode.Uri): string { + return path.basename(path.dirname(uri.fsPath)) || "skill" + } + /** * Scan a skills directory for skill subdirectories. * Handles two symlink cases: @@ -290,6 +307,30 @@ export class SkillsManager { return this.getAllSkills() } + /** + * Get the unique discovered skill names. + */ + getSkillNames(): string[] { + return Array.from(new Set(this.getAllSkills().map((skill) => skill.name))).sort() + } + + /** + * Infer skill provenance for autonomous mutation safety. + * Project and global discovered skills default to user-authored unless + * a higher-level caller tracks agent provenance separately. + */ + getSkillProvenance(name: string): "user" | "bundled" | "hub" | "unknown" { + return this.getAllSkills().some((skill) => skill.name === name) ? "user" : "unknown" + } + + getSkillProvenanceForSource(name: string, source: "global" | "project"): "user" | "bundled" | "hub" | "unknown" { + return this.findSkillByNameAndSource(name, source) ? "user" : "unknown" + } + + hasSkill(name: string, source: "global" | "project"): boolean { + return this.findSkillByNameAndSource(name, source) !== undefined + } + /** * Get a skill by name, source, and optionally mode */ @@ -323,6 +364,36 @@ export class SkillsManager { return { valid: true } } + private validateSkillDocumentStructure(name: string, content: string, expectedDescription?: string): void { + const { data: frontmatter } = matter(content) + const frontmatterName = typeof frontmatter.name === "string" ? frontmatter.name.trim() : "" + const frontmatterDescription = typeof frontmatter.description === "string" ? frontmatter.description.trim() : "" + + if (!frontmatterName || !frontmatterDescription) { + throw new Error( + t("skills:errors.invalid_structure", { + reason: "missing required frontmatter fields: name, description", + }), + ) + } + + if (frontmatterName !== name) { + throw new Error(t("skills:errors.invalid_structure", { reason: `frontmatter name must match \"${name}\"` })) + } + + if (frontmatterDescription.length < 1 || frontmatterDescription.length > 1024) { + throw new Error(t("skills:errors.description_length", { length: frontmatterDescription.length })) + } + + if (expectedDescription && frontmatterDescription !== expectedDescription.trim()) { + throw new Error( + t("skills:errors.invalid_structure", { + reason: "frontmatter description must match the provided description", + }), + ) + } + } + /** * Convert skill name validation error code to a user-friendly error message. */ @@ -350,6 +421,43 @@ export class SkillsManager { source: "global" | "project", description: string, modeSlugs?: string[], + ): Promise { + const titleName = name + .split("-") + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(" ") + + const frontmatterLines = [`name: ${name}`, `description: ${description.trim()}`] + if (modeSlugs && modeSlugs.length > 0) { + frontmatterLines.push(`modeSlugs:`) + for (const slug of modeSlugs) { + frontmatterLines.push(` - ${slug}`) + } + } + + const skillContent = `--- +${frontmatterLines.join("\n")} +--- + +# ${titleName} + +## Instructions + +Add your skill instructions here. +` + + return this.createSkillFromContent(name, source, description, skillContent, modeSlugs) + } + + /** + * Create a new skill from fully prepared SKILL.md content. + */ + async createSkillFromContent( + name: string, + source: "global" | "project", + description: string, + content: string, + modeSlugs?: string[], ): Promise { // Validate skill name const validation = this.validateSkillName(name) @@ -363,6 +471,11 @@ export class SkillsManager { throw new Error(t("skills:errors.description_length", { length: trimmedDescription.length })) } + if (!content.trim()) { + throw new Error(t("skills:errors.description_length", { length: 0 })) + } + this.validateSkillDocumentStructure(name, content, trimmedDescription) + // Determine base directory let baseDir: string if (source === "global") { @@ -375,52 +488,48 @@ export class SkillsManager { baseDir = path.join(provider.cwd, ".roo") } - // Always use the generic skills directory (mode info stored in frontmatter now) const skillsDir = path.join(baseDir, "skills") const skillDir = path.join(skillsDir, name) const skillMdPath = path.join(skillDir, "SKILL.md") - // Check if skill already exists if (await fileExists(skillMdPath)) { throw new Error(t("skills:errors.already_exists", { name, path: skillMdPath })) } - // Create the skill directory await fs.mkdir(skillDir, { recursive: true }) + await fs.writeFile(skillMdPath, content, "utf-8") + await this.discoverSkills() + await this.postSkillsUpdatedMessage(`Skill created: "${name}"`) - // Generate SKILL.md content with frontmatter - const titleName = name - .split("-") - .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) - .join(" ") + return skillMdPath + } - // Build frontmatter with optional modeSlugs - const frontmatterLines = [`name: ${name}`, `description: ${trimmedDescription}`] - if (modeSlugs && modeSlugs.length > 0) { - frontmatterLines.push(`modeSlugs:`) - for (const slug of modeSlugs) { - frontmatterLines.push(` - ${slug}`) - } + /** + * Update the full SKILL.md content for an existing skill. + */ + async updateSkillContent( + name: string, + source: "global" | "project", + content: string, + mode?: string, + ): Promise { + const skill = mode + ? Array.from(this.skills.values()).find( + (candidate) => + candidate.name === name && + candidate.source === source && + (candidate.mode === mode || candidate.modeSlugs?.includes(mode)), + ) + : this.findSkillByNameAndSource(name, source) + if (!skill) { + const modeInfo = mode ? ` (mode: ${mode})` : "" + throw new Error(t("skills:errors.not_found", { name, source, modeInfo })) } + this.validateSkillDocumentStructure(name, content) - const skillContent = `--- -${frontmatterLines.join("\n")} ---- - -# ${titleName} - -## Instructions - -Add your skill instructions here. -` - - // Write the SKILL.md file - await fs.writeFile(skillMdPath, skillContent, "utf-8") - - // Refresh skills list + await fs.writeFile(skill.path, content, "utf-8") await this.discoverSkills() - - return skillMdPath + await this.postSkillsUpdatedMessage(`Skill updated: "${name}"`) } /** @@ -445,6 +554,7 @@ Add your skill instructions here. // Refresh skills list await this.discoverSkills() + await this.postSkillsUpdatedMessage(`Skill deleted: "${name}"`) } /** @@ -516,6 +626,7 @@ Add your skill instructions here. // Refresh skills list await this.discoverSkills() + await this.postSkillsUpdatedMessage(`Skill updated: "${name}"`) } /** @@ -559,6 +670,7 @@ Add your skill instructions here. // Refresh skills list await this.discoverSkills() + await this.postSkillsUpdatedMessage(`Skill updated: "${name}"`) } /** @@ -694,17 +806,23 @@ Add your skill instructions here. watcher.onDidChange(async (uri) => { if (this.isDisposed) return + const skillName = this.getSkillNameFromUri(uri) await this.discoverSkills() + await this.postSkillsUpdatedMessage(`Skill updated: "${skillName}"`) }) watcher.onDidCreate(async (uri) => { if (this.isDisposed) return + const skillName = this.getSkillNameFromUri(uri) await this.discoverSkills() + await this.postSkillsUpdatedMessage(`Skill created: "${skillName}"`) }) watcher.onDidDelete(async (uri) => { if (this.isDisposed) return + const skillName = this.getSkillNameFromUri(uri) await this.discoverSkills() + await this.postSkillsUpdatedMessage(`Skill deleted: "${skillName}"`) }) this.disposables.push(watcher) diff --git a/src/services/skills/__tests__/SkillsManager.spec.ts b/src/services/skills/__tests__/SkillsManager.spec.ts index d36582d893..678267277d 100644 --- a/src/services/skills/__tests__/SkillsManager.spec.ts +++ b/src/services/skills/__tests__/SkillsManager.spec.ts @@ -101,6 +101,7 @@ vi.mock("../../../i18n", () => ({ "skills:errors.name_format": "Skill name must be lowercase letters/numbers/hyphens only (no leading/trailing hyphen, no consecutive hyphens)", "skills:errors.description_length": `Skill description must be 1-1024 characters (got ${params?.length})`, + "skills:errors.invalid_structure": `Invalid SKILL.md structure: ${params?.reason}`, "skills:errors.no_workspace": "Cannot create project skill: no workspace folder is open", "skills:errors.already_exists": `Skill "${params?.name}" already exists at ${params?.path}`, "skills:errors.not_found": `Skill "${params?.name}" not found in ${params?.source}${params?.modeInfo}`, @@ -115,6 +116,7 @@ import { ClineProvider } from "../../../core/webview/ClineProvider" describe("SkillsManager", () => { let skillsManager: SkillsManager let mockProvider: Partial + let mockPostMessageToWebview: ReturnType // Pre-computed paths for tests const globalSkillsDir = p(GLOBAL_ROO_DIR, "skills") @@ -131,10 +133,12 @@ describe("SkillsManager", () => { beforeEach(() => { vi.clearAllMocks() mockHomedir.mockReturnValue(HOME_DIR) + mockPostMessageToWebview = vi.fn() // Create mock provider mockProvider = { cwd: PROJECT_DIR, + postMessageToWebview: mockPostMessageToWebview, customModesManager: { getCustomModes: vi.fn().mockResolvedValue([]), } as any, @@ -1297,6 +1301,212 @@ Instructions`) "already exists", ) }) + + it("should create a skill from provided full content", async () => { + mockDirectoryExists.mockResolvedValue(false) + mockRealpath.mockImplementation(async (p: string) => p) + mockReaddir.mockResolvedValue([]) + mockFileExists.mockResolvedValue(false) + mockMkdir.mockResolvedValue(undefined) + mockWriteFile.mockResolvedValue(undefined) + + const content = `---\nname: workflow-read-file-search-files\ndescription: Use when read_file and search_files succeed repeatedly.\n---\n\n# Workflow\n\nUse it.` + + const createdPath = await skillsManager.createSkillFromContent( + "workflow-read-file-search-files", + "project", + "Use when read_file and search_files succeed repeatedly.", + content, + ["code"], + ) + + expect(createdPath).toBe(p(PROJECT_DIR, ".roo", "skills", "workflow-read-file-search-files", "SKILL.md")) + expect(mockWriteFile).toHaveBeenCalledWith( + p(PROJECT_DIR, ".roo", "skills", "workflow-read-file-search-files", "SKILL.md"), + content, + "utf-8", + ) + }) + + it("rejects createSkillFromContent when SKILL.md frontmatter is missing required fields", async () => { + mockDirectoryExists.mockResolvedValue(false) + mockFileExists.mockResolvedValue(false) + + await expect( + skillsManager.createSkillFromContent( + "workflow-read-file-search-files", + "project", + "Use when read_file and search_files succeed repeatedly.", + "# Workflow\n\nUse it.", + ["code"], + ), + ).rejects.toThrow("Invalid SKILL.md structure") + expect(mockWriteFile).not.toHaveBeenCalled() + }) + + it("should push a live skills created message to the webview after creating a skill", async () => { + const newSkillDir = p(globalSkillsDir, "new-skill") + const newSkillMd = p(newSkillDir, "SKILL.md") + let created = false + + mockDirectoryExists.mockImplementation(async (dir: string) => created && dir === globalSkillsDir) + mockRealpath.mockImplementation(async (p: string) => p) + mockReaddir.mockImplementation(async (dir: string) => + created && dir === globalSkillsDir ? ["new-skill"] : [], + ) + mockStat.mockImplementation(async (pathArg: string) => { + if (created && pathArg === newSkillDir) { + return { isDirectory: () => true } + } + throw new Error("Not found") + }) + mockFileExists.mockImplementation(async (file: string) => created && file === newSkillMd) + mockMkdir.mockResolvedValue(undefined) + mockWriteFile.mockImplementation(async () => { + created = true + }) + mockReadFile.mockImplementation(async (file: string) => { + if (created && file === newSkillMd) { + return `---\nname: new-skill\ndescription: A new skill description\n---\n\n# New Skill` + } + throw new Error("File not found") + }) + + await skillsManager.createSkill("new-skill", "global", "A new skill description") + + expect(mockPostMessageToWebview).toHaveBeenCalledWith( + expect.objectContaining({ + type: "skillsUpdated", + text: expect.stringContaining("new-skill"), + skills: expect.arrayContaining([ + expect.objectContaining({ + name: "new-skill", + source: "global", + }), + ]), + }), + ) + }) + }) + + describe("updateSkillContent", () => { + it("should update an existing skill with new content", async () => { + const testSkillDir = p(globalSkillsDir, "test-skill") + const testSkillMd = p(testSkillDir, "SKILL.md") + + mockDirectoryExists.mockImplementation(async (dir: string) => dir === globalSkillsDir) + mockRealpath.mockImplementation(async (pathArg: string) => pathArg) + mockReaddir.mockImplementation(async (dir: string) => (dir === globalSkillsDir ? ["test-skill"] : [])) + mockStat.mockImplementation(async (pathArg: string) => { + if (pathArg === testSkillDir) { + return { isDirectory: () => true } + } + throw new Error("Not found") + }) + mockFileExists.mockImplementation(async (file: string) => file === testSkillMd) + mockReadFile.mockResolvedValue(`---\nname: test-skill\ndescription: A test skill\n---\n\nOriginal content`) + mockWriteFile.mockResolvedValue(undefined) + + await skillsManager.discoverSkills() + + const updatedContent = `---\nname: test-skill\ndescription: Updated test skill\n---\n\nUpdated content` + + await expect( + skillsManager.updateSkillContent("test-skill", "global", updatedContent), + ).resolves.toBeUndefined() + expect(mockWriteFile).toHaveBeenCalledWith(testSkillMd, updatedContent, "utf-8") + }) + + it("rejects updateSkillContent when SKILL.md frontmatter becomes invalid", async () => { + const testSkillDir = p(globalSkillsDir, "test-skill") + const testSkillMd = p(testSkillDir, "SKILL.md") + + mockDirectoryExists.mockImplementation(async (dir: string) => dir === globalSkillsDir) + mockRealpath.mockImplementation(async (pathArg: string) => pathArg) + mockReaddir.mockImplementation(async (dir: string) => (dir === globalSkillsDir ? ["test-skill"] : [])) + mockStat.mockImplementation(async (pathArg: string) => { + if (pathArg === testSkillDir) { + return { isDirectory: () => true } + } + throw new Error("Not found") + }) + mockFileExists.mockImplementation(async (file: string) => file === testSkillMd) + mockReadFile.mockResolvedValue(`---\nname: test-skill\ndescription: A test skill\n---\n\nOriginal content`) + mockWriteFile.mockResolvedValue(undefined) + + await skillsManager.discoverSkills() + + await expect(skillsManager.updateSkillContent("test-skill", "global", "# Broken content")).rejects.toThrow( + "Invalid SKILL.md structure", + ) + expect(mockWriteFile).not.toHaveBeenCalled() + }) + + it("updates a multi-mode skill when addressed by a secondary mode slug", async () => { + const testSkillDir = p(globalSkillsDir, "test-skill") + const testSkillMd = p(testSkillDir, "SKILL.md") + + mockDirectoryExists.mockImplementation(async (dir: string) => dir === globalSkillsDir) + mockRealpath.mockImplementation(async (pathArg: string) => pathArg) + mockReaddir.mockImplementation(async (dir: string) => (dir === globalSkillsDir ? ["test-skill"] : [])) + mockStat.mockImplementation(async (pathArg: string) => { + if (pathArg === testSkillDir) { + return { isDirectory: () => true } + } + throw new Error("Not found") + }) + mockFileExists.mockImplementation(async (file: string) => file === testSkillMd) + mockReadFile.mockResolvedValue( + `---\nname: test-skill\ndescription: A test skill\nmodeSlugs:\n - code\n - architect\n---\n\nOriginal content`, + ) + mockWriteFile.mockResolvedValue(undefined) + + await skillsManager.discoverSkills() + + const updatedContent = `---\nname: test-skill\ndescription: Updated test skill\nmodeSlugs:\n - code\n - architect\n---\n\nUpdated content` + + await expect( + skillsManager.updateSkillContent("test-skill", "global", updatedContent, "architect"), + ).resolves.toBeUndefined() + expect(mockWriteFile).toHaveBeenCalledWith(testSkillMd, updatedContent, "utf-8") + }) + + it("should push a live skills updated message to the webview after updating a skill", async () => { + const testSkillDir = p(globalSkillsDir, "test-skill") + const testSkillMd = p(testSkillDir, "SKILL.md") + + mockDirectoryExists.mockImplementation(async (dir: string) => dir === globalSkillsDir) + mockRealpath.mockImplementation(async (pathArg: string) => pathArg) + mockReaddir.mockImplementation(async (dir: string) => (dir === globalSkillsDir ? ["test-skill"] : [])) + mockStat.mockImplementation(async (pathArg: string) => { + if (pathArg === testSkillDir) { + return { isDirectory: () => true } + } + throw new Error("Not found") + }) + mockFileExists.mockImplementation(async (file: string) => file === testSkillMd) + mockReadFile.mockResolvedValue(`---\nname: test-skill\ndescription: A test skill\n---\n\nOriginal content`) + mockWriteFile.mockResolvedValue(undefined) + + await skillsManager.discoverSkills() + + const updatedContent = `---\nname: test-skill\ndescription: Updated test skill\n---\n\nUpdated content` + + await skillsManager.updateSkillContent("test-skill", "global", updatedContent) + + expect(mockPostMessageToWebview).toHaveBeenCalledWith( + expect.objectContaining({ + type: "skillsUpdated", + text: expect.stringContaining("test-skill"), + skills: expect.arrayContaining([ + expect.objectContaining({ + name: "test-skill", + source: "global", + }), + ]), + }), + ) + }) }) describe("deleteSkill", () => { diff --git a/src/shared/__tests__/experiments.spec.ts b/src/shared/__tests__/experiments.spec.ts index 92a7d7604f..9a6a2c0af7 100644 --- a/src/shared/__tests__/experiments.spec.ts +++ b/src/shared/__tests__/experiments.spec.ts @@ -14,6 +14,24 @@ describe("experiments", () => { }) }) + describe("SELF_IMPROVING", () => { + it("is configured correctly", () => { + expect(EXPERIMENT_IDS.SELF_IMPROVING).toBe("selfImproving") + expect(experimentConfigsMap.SELF_IMPROVING).toMatchObject({ + enabled: false, + }) + }) + }) + + describe("SELF_IMPROVING_AUTO_SKILLS", () => { + it("is configured correctly", () => { + expect(EXPERIMENT_IDS.SELF_IMPROVING_AUTO_SKILLS).toBe("selfImprovingAutoSkills") + expect(experimentConfigsMap.SELF_IMPROVING_AUTO_SKILLS).toMatchObject({ + enabled: false, + }) + }) + }) + describe("isEnabled", () => { it("returns false when experiment is not enabled", () => { const experiments: Record = { @@ -21,6 +39,8 @@ describe("experiments", () => { imageGeneration: false, runSlashCommand: false, customTools: false, + selfImproving: false, + selfImprovingAutoSkills: false, } expect(Experiments.isEnabled(experiments, EXPERIMENT_IDS.PREVENT_FOCUS_DISRUPTION)).toBe(false) }) @@ -31,6 +51,8 @@ describe("experiments", () => { imageGeneration: false, runSlashCommand: false, customTools: false, + selfImproving: false, + selfImprovingAutoSkills: false, } expect(Experiments.isEnabled(experiments, EXPERIMENT_IDS.PREVENT_FOCUS_DISRUPTION)).toBe(true) }) @@ -41,8 +63,23 @@ describe("experiments", () => { imageGeneration: false, runSlashCommand: false, customTools: false, + selfImproving: false, + selfImprovingAutoSkills: false, } expect(Experiments.isEnabled(experiments, EXPERIMENT_IDS.PREVENT_FOCUS_DISRUPTION)).toBe(false) }) + + it("returns false for self improving by default", () => { + const experiments: Record = { + preventFocusDisruption: false, + imageGeneration: false, + runSlashCommand: false, + customTools: false, + selfImproving: false, + selfImprovingAutoSkills: false, + } + + expect(Experiments.isEnabled(experiments, EXPERIMENT_IDS.SELF_IMPROVING)).toBe(false) + }) }) }) diff --git a/src/shared/experiments.ts b/src/shared/experiments.ts index e189f99e23..2e4a5dd7b0 100644 --- a/src/shared/experiments.ts +++ b/src/shared/experiments.ts @@ -5,6 +5,8 @@ export const EXPERIMENT_IDS = { IMAGE_GENERATION: "imageGeneration", RUN_SLASH_COMMAND: "runSlashCommand", CUSTOM_TOOLS: "customTools", + SELF_IMPROVING: "selfImproving", + SELF_IMPROVING_AUTO_SKILLS: "selfImprovingAutoSkills", } as const satisfies Record type _AssertExperimentIds = AssertEqual>> @@ -20,6 +22,8 @@ export const experimentConfigsMap: Record = { IMAGE_GENERATION: { enabled: false }, RUN_SLASH_COMMAND: { enabled: false }, CUSTOM_TOOLS: { enabled: false }, + SELF_IMPROVING: { enabled: false }, + SELF_IMPROVING_AUTO_SKILLS: { enabled: false }, } export const experimentDefault = Object.fromEntries( diff --git a/webview-ui/src/components/settings/ExperimentalFeature.tsx b/webview-ui/src/components/settings/ExperimentalFeature.tsx index a96a00a426..8c427308a5 100644 --- a/webview-ui/src/components/settings/ExperimentalFeature.tsx +++ b/webview-ui/src/components/settings/ExperimentalFeature.tsx @@ -6,9 +6,10 @@ interface ExperimentalFeatureProps { onChange: (value: boolean) => void // Additional property to identify the experiment experimentKey?: string + checkboxTestId?: string } -export const ExperimentalFeature = ({ enabled, onChange, experimentKey }: ExperimentalFeatureProps) => { +export const ExperimentalFeature = ({ enabled, onChange, experimentKey, checkboxTestId }: ExperimentalFeatureProps) => { const { t } = useAppTranslation() // Generate translation keys based on experiment key @@ -18,7 +19,10 @@ export const ExperimentalFeature = ({ enabled, onChange, experimentKey }: Experi return (
- onChange(e.target.checked)}> + onChange(e.target.checked)} + data-testid={checkboxTestId}> {t(nameKey)}
diff --git a/webview-ui/src/components/settings/ExperimentalSettings.tsx b/webview-ui/src/components/settings/ExperimentalSettings.tsx index 23786ce0b9..a02c135b53 100644 --- a/webview-ui/src/components/settings/ExperimentalSettings.tsx +++ b/webview-ui/src/components/settings/ExperimentalSettings.tsx @@ -5,6 +5,7 @@ import type { Experiments, ImageGenerationProvider } from "@roo-code/types" import { EXPERIMENT_IDS, experimentConfigsMap } from "@roo/experiments" import { useAppTranslation } from "@src/i18n/TranslationContext" +import { Input, Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui" import { cn } from "@src/lib/utils" import { SetExperimentEnabled } from "./types" @@ -23,9 +24,17 @@ type ExperimentalSettingsProps = HTMLAttributes & { imageGenerationProvider?: ImageGenerationProvider openRouterImageApiKey?: string openRouterImageGenerationSelectedModel?: string + memoryBackend?: "builtin" | "agentmemory" + agentMemoryUrl?: string + selfImprovingScope?: "workspace" | "global" + selfImprovingAutoSkillsScope?: "workspace" | "global" setImageGenerationProvider?: (provider: ImageGenerationProvider) => void setOpenRouterImageApiKey?: (apiKey: string) => void setImageGenerationSelectedModel?: (model: string) => void + setMemoryBackend?: (backend: "builtin" | "agentmemory") => void + setAgentMemoryUrl?: (url: string) => void + setSelfImprovingScope?: (scope: "workspace" | "global") => void + setSelfImprovingAutoSkillsScope?: (scope: "workspace" | "global") => void } export const ExperimentalSettings = ({ @@ -36,13 +45,26 @@ export const ExperimentalSettings = ({ imageGenerationProvider, openRouterImageApiKey, openRouterImageGenerationSelectedModel, + memoryBackend, + agentMemoryUrl, + selfImprovingScope, + selfImprovingAutoSkillsScope, setImageGenerationProvider, setOpenRouterImageApiKey, setImageGenerationSelectedModel, + setMemoryBackend, + setAgentMemoryUrl, + setSelfImprovingScope, + setSelfImprovingAutoSkillsScope, className, ...props }: ExperimentalSettingsProps) => { const { t } = useAppTranslation() + const autoSkillsVisible = experiments[EXPERIMENT_IDS.SELF_IMPROVING] ?? false + const autoSkillsEnabled = experiments[EXPERIMENT_IDS.SELF_IMPROVING_AUTO_SKILLS] ?? false + const currentMemoryBackend = memoryBackend ?? "builtin" + const currentSelfImprovingScope = selfImprovingScope ?? "global" + const currentAutoSkillsScope = selfImprovingAutoSkillsScope ?? "workspace" return (
@@ -50,9 +72,8 @@ export const ExperimentalSettings = ({
{Object.entries(experimentConfigsMap) - .filter(([key]) => key in EXPERIMENT_IDS) + .filter(([key]) => key in EXPERIMENT_IDS && key !== "SELF_IMPROVING_AUTO_SKILLS") .map((config) => { - // Use the same translation key pattern as ExperimentalFeature const experimentKey = config[0] const label = t(`settings:experimental.${experimentKey}.name`) @@ -99,6 +120,177 @@ export const ExperimentalSettings = ({ ) } + if (config[0] === "SELF_IMPROVING") { + return ( + +
+ + setExperimentEnabled(EXPERIMENT_IDS.SELF_IMPROVING, enabled) + } + checkboxTestId="experimental-self-improving-checkbox" + /> + {autoSkillsVisible && ( +
+ {setSelfImprovingScope && ( +
+ + +
+ )} + + setExperimentEnabled( + EXPERIMENT_IDS.SELF_IMPROVING_AUTO_SKILLS, + enabled, + ) + } + checkboxTestId="experimental-self-improving-auto-skills-checkbox" + /> + {autoSkillsEnabled && setSelfImprovingAutoSkillsScope && ( +
+ + +
+ )} + {setMemoryBackend && ( +
+ + +
+ )} + {currentMemoryBackend === "agentmemory" && setAgentMemoryUrl && ( +
+ + setAgentMemoryUrl(event.target.value)} + placeholder="http://localhost:3111" + data-testid="self-improving-agent-memory-url-input" + /> +
+ )} +
+ )} +
+ + ) + } return ( (({ onDone, t imageGenerationProvider, openRouterImageApiKey, openRouterImageGenerationSelectedModel, + memoryBackend, + agentMemoryUrl, + selfImprovingScope, + selfImprovingAutoSkillsScope, reasoningBlockCollapsed, enterBehavior, includeCurrentTime, @@ -342,6 +346,50 @@ const SettingsView = forwardRef(({ onDone, t }) }, []) + const setMemoryBackend = useCallback((backend: "builtin" | "agentmemory") => { + setCachedState((prevState) => { + if (prevState.memoryBackend === backend) { + return prevState + } + + setChangeDetected(true) + return { ...prevState, memoryBackend: backend } + }) + }, []) + + const setAgentMemoryUrl = useCallback((url: string) => { + setCachedState((prevState) => { + if (prevState.agentMemoryUrl === url) { + return prevState + } + + setChangeDetected(true) + return { ...prevState, agentMemoryUrl: url } + }) + }, []) + + const setSelfImprovingScope = useCallback((scope: "workspace" | "global") => { + setCachedState((prevState) => { + if (prevState.selfImprovingScope === scope) { + return prevState + } + + setChangeDetected(true) + return { ...prevState, selfImprovingScope: scope } + }) + }, []) + + const setSelfImprovingAutoSkillsScope = useCallback((scope: "workspace" | "global") => { + setCachedState((prevState) => { + if (prevState.selfImprovingAutoSkillsScope === scope) { + return prevState + } + + setChangeDetected(true) + return { ...prevState, selfImprovingAutoSkillsScope: scope } + }) + }, []) + const setCustomSupportPromptsField = useCallback((prompts: Record) => { setCachedState((prevState) => { const previousStr = JSON.stringify(prevState.customSupportPrompts) @@ -420,6 +468,10 @@ const SettingsView = forwardRef(({ onDone, t imageGenerationProvider, openRouterImageApiKey, openRouterImageGenerationSelectedModel, + memoryBackend, + agentMemoryUrl: agentMemoryUrl || "http://localhost:3111", + selfImprovingScope: selfImprovingScope ?? "global", + selfImprovingAutoSkillsScope: selfImprovingAutoSkillsScope ?? "workspace", experiments, customSupportPrompts, }, @@ -904,13 +956,19 @@ const SettingsView = forwardRef(({ onDone, t apiConfiguration={apiConfiguration} setApiConfigurationField={setApiConfigurationField} imageGenerationProvider={imageGenerationProvider} - openRouterImageApiKey={openRouterImageApiKey as string | undefined} - openRouterImageGenerationSelectedModel={ - openRouterImageGenerationSelectedModel as string | undefined - } + openRouterImageApiKey={openRouterImageApiKey} + openRouterImageGenerationSelectedModel={openRouterImageGenerationSelectedModel} + memoryBackend={memoryBackend} + agentMemoryUrl={agentMemoryUrl} + selfImprovingScope={selfImprovingScope} + selfImprovingAutoSkillsScope={selfImprovingAutoSkillsScope} setImageGenerationProvider={setImageGenerationProvider} setOpenRouterImageApiKey={setOpenRouterImageApiKey} setImageGenerationSelectedModel={setImageGenerationSelectedModel} + setMemoryBackend={setMemoryBackend} + setAgentMemoryUrl={setAgentMemoryUrl} + setSelfImprovingScope={setSelfImprovingScope} + setSelfImprovingAutoSkillsScope={setSelfImprovingAutoSkillsScope} /> )} diff --git a/webview-ui/src/components/settings/SkillsSettings.tsx b/webview-ui/src/components/settings/SkillsSettings.tsx index bf81d0c6c7..cd56c87100 100644 --- a/webview-ui/src/components/settings/SkillsSettings.tsx +++ b/webview-ui/src/components/settings/SkillsSettings.tsx @@ -35,12 +35,13 @@ import { CreateSkillDialog } from "./CreateSkillDialog" export const SkillsSettings: React.FC = () => { const { t } = useAppTranslation() - const { cwd, skills: rawSkills, customModes } = useExtensionState() + const { cwd, skills: rawSkills, customModes, skillsUpdateNotice } = useExtensionState() const skills = useMemo(() => rawSkills ?? [], [rawSkills]) const [deleteDialogOpen, setDeleteDialogOpen] = useState(false) const [skillToDelete, setSkillToDelete] = useState(null) const [createDialogOpen, setCreateDialogOpen] = useState(false) + const [visibleSkillsUpdateNotice, setVisibleSkillsUpdateNotice] = useState() // Mode selection modal state const [modeDialogOpen, setModeDialogOpen] = useState(false) @@ -65,6 +66,20 @@ export const SkillsSettings: React.FC = () => { handleRefresh() }, [handleRefresh]) + useEffect(() => { + if (!skillsUpdateNotice) { + setVisibleSkillsUpdateNotice(undefined) + return + } + + setVisibleSkillsUpdateNotice(skillsUpdateNotice) + const timeoutId = window.setTimeout(() => { + setVisibleSkillsUpdateNotice(undefined) + }, 3500) + + return () => window.clearTimeout(timeoutId) + }, [skillsUpdateNotice]) + const handleDeleteClick = useCallback((skill: SkillMetadata) => { setSkillToDelete(skill) setDeleteDialogOpen(true) @@ -238,6 +253,15 @@ export const SkillsSettings: React.FC = () => { {t("settings:skills.addSkill")} + + {visibleSkillsUpdateNotice && ( +
+ {visibleSkillsUpdateNotice} +
+ )}
diff --git a/webview-ui/src/components/settings/__tests__/SettingsView.spec.tsx b/webview-ui/src/components/settings/__tests__/SettingsView.spec.tsx index 6fa34ebe81..d99087851a 100644 --- a/webview-ui/src/components/settings/__tests__/SettingsView.spec.tsx +++ b/webview-ui/src/components/settings/__tests__/SettingsView.spec.tsx @@ -173,21 +173,17 @@ vi.mock("@/components/ui", () => ({ Input: ({ value, onChange, placeholder, "data-testid": dataTestId }: any) => ( ), - Select: ({ children, value, onValueChange }: any) => ( -
- + Select: ({ children, value, onValueChange, "data-testid": dataTestId }: any) => ( + ), - SelectTrigger: ({ children }: any) =>
{children}
, - SelectValue: ({ placeholder }: any) =>
{placeholder}
, + SelectContent: ({ children }: any) => <>{children}, + SelectGroup: ({ children }: any) => <>{children}, + SelectItem: ({ children, value }: any) => , + SelectTrigger: () => null, + SelectValue: () => null, SearchableSelect: ({ value, onValueChange, options, placeholder }: any) => (