diff --git a/extensions/cli/src/services/ModelService.test.ts b/extensions/cli/src/services/ModelService.test.ts index 1425ad55069..edaf4b19a80 100644 --- a/extensions/cli/src/services/ModelService.test.ts +++ b/extensions/cli/src/services/ModelService.test.ts @@ -9,7 +9,7 @@ import * as workos from "../auth/workos.js"; import { AuthConfig } from "../auth/workos.js"; import * as config from "../config.js"; -import { ModelService } from "./ModelService.js"; +import { ModelService, expandAutodetectModels } from "./ModelService.js"; describe("ModelService", () => { let service: ModelService; @@ -366,4 +366,194 @@ describe("ModelService", () => { expect(errorListener).toHaveBeenCalledWith(expect.any(Error)); }); }); + + describe("expandAutodetectModels()", () => { + test("expands AUTODETECT into concrete models", async () => { + const autodetectModel: ModelConfig = { + provider: "ollama", + model: "AUTODETECT", + apiBase: "http://localhost:11434", + roles: ["chat"], + } as ModelConfig; + + const mockLlmApi = { + list: vi.fn().mockResolvedValue([ + { id: "llama3", object: "model", owned_by: "ollama" }, + { id: "mistral", object: "model", owned_by: "ollama" }, + ]), + }; + + vi.mocked(config.createLlmApi).mockReturnValue(mockLlmApi as any); + + const result = await expandAutodetectModels( + [autodetectModel], + mockAuthConfig, + ); + + expect(result).toHaveLength(2); + expect(result[0]).toEqual({ + ...autodetectModel, + model: "llama3", + name: "llama3", + }); + expect(result[1]).toEqual({ + ...autodetectModel, + model: "mistral", + name: "mistral", + }); + }); + + test("passes through non-AUTODETECT models unchanged", async () => { + const gpt4Model: ModelConfig = { + provider: "openai", + model: "gpt-4", + name: "GPT-4", + apiKey: "test-key", + roles: ["chat"], + } as ModelConfig; + + vi.mocked(config.createLlmApi).mockReturnValue(mockLlmApi as any); + + const result = await expandAutodetectModels([gpt4Model], mockAuthConfig); + + expect(result).toHaveLength(1); + expect(result[0]).toBe(gpt4Model); + }); + + test("handles list() failure gracefully", async () => { + const autodetectModel: ModelConfig = { + provider: "ollama", + model: "AUTODETECT", + apiBase: "http://localhost:11434", + roles: ["chat"], + } as ModelConfig; + + const mockLlmApi = { + list: vi.fn().mockRejectedValue(new Error("Connection failed")), + }; + + vi.mocked(config.createLlmApi).mockReturnValue(mockLlmApi as any); + + const result = await expandAutodetectModels( + [autodetectModel], + mockAuthConfig, + ); + + // AUTODETECT entry should be skipped when list() fails + expect(result).toHaveLength(0); + }); + + test("handles empty list() results", async () => { + const autodetectModel: ModelConfig = { + provider: "ollama", + model: "AUTODETECT", + apiBase: "http://localhost:11434", + roles: ["chat"], + } as ModelConfig; + + const mockLlmApi = { + list: vi.fn().mockResolvedValue([]), + }; + + vi.mocked(config.createLlmApi).mockReturnValue(mockLlmApi as any); + + const result = await expandAutodetectModels( + [autodetectModel], + mockAuthConfig, + ); + + // AUTODETECT entry should be skipped when no models are returned + expect(result).toHaveLength(0); + }); + + test("filters out AUTODETECT from list results", async () => { + const autodetectModel: ModelConfig = { + provider: "ollama", + model: "AUTODETECT", + apiBase: "http://localhost:11434", + roles: ["chat"], + } as ModelConfig; + + const mockLlmApi = { + list: vi.fn().mockResolvedValue([ + { id: "AUTODETECT", object: "model", owned_by: "ollama" }, + { id: "real-model", object: "model", owned_by: "ollama" }, + ]), + }; + + vi.mocked(config.createLlmApi).mockReturnValue(mockLlmApi as any); + + const result = await expandAutodetectModels( + [autodetectModel], + mockAuthConfig, + ); + + // Only real-model should appear, AUTODETECT should be filtered + expect(result).toHaveLength(1); + expect(result[0]).toEqual({ + ...autodetectModel, + model: "real-model", + name: "real-model", + }); + }); + + test("handles mixed AUTODETECT and non-AUTODETECT models", async () => { + const autodetectModel: ModelConfig = { + provider: "ollama", + model: "AUTODETECT", + apiBase: "http://localhost:11434", + roles: ["chat"], + } as ModelConfig; + + const gpt4Model: ModelConfig = { + provider: "openai", + model: "gpt-4", + name: "GPT-4", + apiKey: "test-key", + roles: ["chat"], + } as ModelConfig; + + const mockLlmApi = { + list: vi + .fn() + .mockResolvedValue([ + { id: "llama3", object: "model", owned_by: "ollama" }, + ]), + }; + + vi.mocked(config.createLlmApi).mockReturnValue(mockLlmApi as any); + + const result = await expandAutodetectModels( + [autodetectModel, gpt4Model], + mockAuthConfig, + ); + + // Should have the expanded llama3 and the unchanged gpt-4 + expect(result).toHaveLength(2); + expect(result[0]).toEqual({ + ...autodetectModel, + model: "llama3", + name: "llama3", + }); + expect(result[1]).toBe(gpt4Model); + }); + + test("handles createLlmApi returning null", async () => { + const autodetectModel: ModelConfig = { + provider: "invalid-provider", + model: "AUTODETECT", + roles: ["chat"], + } as ModelConfig; + + vi.mocked(config.createLlmApi).mockReturnValue(null as any); + + const result = await expandAutodetectModels( + [autodetectModel], + mockAuthConfig, + ); + + // AUTODETECT entry should be skipped when createLlmApi returns null + expect(result).toHaveLength(0); + }); + }); }); diff --git a/extensions/cli/src/services/ModelService.ts b/extensions/cli/src/services/ModelService.ts index 9e542205707..5da30467ef8 100644 --- a/extensions/cli/src/services/ModelService.ts +++ b/extensions/cli/src/services/ModelService.ts @@ -7,6 +7,65 @@ import { logger } from "../util/logger.js"; import { BaseService, ServiceWithDependencies } from "./BaseService.js"; import { AgentFileServiceState, ModelServiceState } from "./types.js"; +const AUTODETECT = "AUTODETECT"; + +/** + * Expands AUTODETECT model entries into concrete models + * For each model with model === "AUTODETECT", calls llm.list() to get available models + * and replaces the AUTODETECT entry with one ModelConfig per available model. + * Handles errors gracefully by skipping failed expansions with a warning. + */ +export async function expandAutodetectModels( + models: ModelConfig[], + authConfig: AuthConfig, +): Promise { + const expanded: ModelConfig[] = []; + + for (const model of models) { + if (model.model !== AUTODETECT) { + expanded.push(model); + continue; + } + + const llmApi = createLlmApi(model, authConfig); + if (!llmApi) { + logger.warn( + `Cannot create API client for AUTODETECT model with provider ${model.provider}, skipping`, + ); + continue; + } + + try { + const availableModels = await llmApi.list(); + if (!availableModels || availableModels.length === 0) { + logger.warn( + `No models returned from ${model.provider} for AUTODETECT, skipping`, + ); + continue; + } + + for (const available of availableModels) { + if (available.id === AUTODETECT) continue; + expanded.push({ + ...model, + model: available.id, + name: available.id, + }); + } + + logger.debug( + `Expanded AUTODETECT for ${model.provider}: ${availableModels.map((m) => m.id).join(", ")}`, + ); + } catch (e) { + logger.warn( + `Error listing models for AUTODETECT expansion (${model.provider}): ${e}`, + ); + } + } + + return expanded; +} + /** * Service for managing LLM and model state * Depends on auth config and assistant config @@ -49,8 +108,20 @@ export class ModelService assistantModelsCount: assistant?.models?.length || 0, }); - this.assistant = assistant; this.authConfig = authConfig; + + // Expand AUTODETECT model entries into concrete models + if (assistant.models?.some((m) => m?.model === AUTODETECT)) { + assistant = { + ...assistant, + models: await expandAutodetectModels( + assistant.models.filter((m): m is ModelConfig => !!m), + authConfig, + ), + }; + } + + this.assistant = assistant; this.availableModels = (assistant.models?.filter( (model) => model && (model.roles?.includes("chat") || model.roles === undefined),