fix(cli): expand AUTODETECT model entries before sending API requests

extensions/cli/src/services/ModelService.test.ts

@@ -9,7 +9,7 @@ import * as workos from "../auth/workos.js";
 import { AuthConfig } from "../auth/workos.js";
 import * as config from "../config.js";
 
-import { ModelService } from "./ModelService.js";
+import { ModelService, expandAutodetectModels } from "./ModelService.js";
 
 describe("ModelService", () => {
   let service: ModelService;
@@ -366,4 +366,194 @@ describe("ModelService", () => {
       expect(errorListener).toHaveBeenCalledWith(expect.any(Error));
     });
   });
+
+  describe("expandAutodetectModels()", () => {
+    test("expands AUTODETECT into concrete models", async () => {
+      const autodetectModel: ModelConfig = {
+        provider: "ollama",
+        model: "AUTODETECT",
+        apiBase: "http://localhost:11434",
+        roles: ["chat"],
+      } as ModelConfig;
+
+      const mockLlmApi = {
+        list: vi.fn().mockResolvedValue([
+          { id: "llama3", object: "model", owned_by: "ollama" },
+          { id: "mistral", object: "model", owned_by: "ollama" },
+        ]),
+      };
+
+      vi.mocked(config.createLlmApi).mockReturnValue(mockLlmApi as any);
+
+      const result = await expandAutodetectModels(
+        [autodetectModel],
+        mockAuthConfig,
+      );
+
+      expect(result).toHaveLength(2);
+      expect(result[0]).toEqual({
+        ...autodetectModel,
+        model: "llama3",
+        name: "llama3",
+      });
+      expect(result[1]).toEqual({
+        ...autodetectModel,
+        model: "mistral",
+        name: "mistral",
+      });
+    });
+
+    test("passes through non-AUTODETECT models unchanged", async () => {
+      const gpt4Model: ModelConfig = {
+        provider: "openai",
+        model: "gpt-4",
+        name: "GPT-4",
+        apiKey: "test-key",
+        roles: ["chat"],
+      } as ModelConfig;
+
+      vi.mocked(config.createLlmApi).mockReturnValue(mockLlmApi as any);
+
+      const result = await expandAutodetectModels([gpt4Model], mockAuthConfig);
+
+      expect(result).toHaveLength(1);
+      expect(result[0]).toBe(gpt4Model);
+    });
+
+    test("handles list() failure gracefully", async () => {
+      const autodetectModel: ModelConfig = {
+        provider: "ollama",
+        model: "AUTODETECT",
+        apiBase: "http://localhost:11434",
+        roles: ["chat"],
+      } as ModelConfig;
+
+      const mockLlmApi = {
+        list: vi.fn().mockRejectedValue(new Error("Connection failed")),
+      };
+
+      vi.mocked(config.createLlmApi).mockReturnValue(mockLlmApi as any);
+
+      const result = await expandAutodetectModels(
+        [autodetectModel],
+        mockAuthConfig,
+      );
+
+      // AUTODETECT entry should be skipped when list() fails
+      expect(result).toHaveLength(0);
+    });
+
+    test("handles empty list() results", async () => {
+      const autodetectModel: ModelConfig = {
+        provider: "ollama",
+        model: "AUTODETECT",
+        apiBase: "http://localhost:11434",
+        roles: ["chat"],
+      } as ModelConfig;
+
+      const mockLlmApi = {
+        list: vi.fn().mockResolvedValue([]),
+      };
+
+      vi.mocked(config.createLlmApi).mockReturnValue(mockLlmApi as any);
+
+      const result = await expandAutodetectModels(
+        [autodetectModel],
+        mockAuthConfig,
+      );
+
+      // AUTODETECT entry should be skipped when no models are returned
+      expect(result).toHaveLength(0);
+    });
+
+    test("filters out AUTODETECT from list results", async () => {
+      const autodetectModel: ModelConfig = {
+        provider: "ollama",
+        model: "AUTODETECT",
+        apiBase: "http://localhost:11434",
+        roles: ["chat"],
+      } as ModelConfig;
+
+      const mockLlmApi = {
+        list: vi.fn().mockResolvedValue([
+          { id: "AUTODETECT", object: "model", owned_by: "ollama" },
+          { id: "real-model", object: "model", owned_by: "ollama" },
+        ]),
+      };
+
+      vi.mocked(config.createLlmApi).mockReturnValue(mockLlmApi as any);
+
+      const result = await expandAutodetectModels(
+        [autodetectModel],
+        mockAuthConfig,
+      );
+
+      // Only real-model should appear, AUTODETECT should be filtered
+      expect(result).toHaveLength(1);
+      expect(result[0]).toEqual({
+        ...autodetectModel,
+        model: "real-model",
+        name: "real-model",
+      });
+    });
+
+    test("handles mixed AUTODETECT and non-AUTODETECT models", async () => {
+      const autodetectModel: ModelConfig = {
+        provider: "ollama",
+        model: "AUTODETECT",
+        apiBase: "http://localhost:11434",
+        roles: ["chat"],
+      } as ModelConfig;
+
+      const gpt4Model: ModelConfig = {
+        provider: "openai",
+        model: "gpt-4",
+        name: "GPT-4",
+        apiKey: "test-key",
+        roles: ["chat"],
+      } as ModelConfig;
+
+      const mockLlmApi = {
+        list: vi
+          .fn()
+          .mockResolvedValue([
+            { id: "llama3", object: "model", owned_by: "ollama" },
+          ]),
+      };
+
+      vi.mocked(config.createLlmApi).mockReturnValue(mockLlmApi as any);
+
+      const result = await expandAutodetectModels(
+        [autodetectModel, gpt4Model],
+        mockAuthConfig,
+      );
+
+      // Should have the expanded llama3 and the unchanged gpt-4
+      expect(result).toHaveLength(2);
+      expect(result[0]).toEqual({
+        ...autodetectModel,
+        model: "llama3",
+        name: "llama3",
+      });
+      expect(result[1]).toBe(gpt4Model);
+    });
+
+    test("handles createLlmApi returning null", async () => {
+      const autodetectModel: ModelConfig = {
+        provider: "invalid-provider",
+        model: "AUTODETECT",
+        roles: ["chat"],
+      } as ModelConfig;
+
+      vi.mocked(config.createLlmApi).mockReturnValue(null as any);
+
+      const result = await expandAutodetectModels(
+        [autodetectModel],
+        mockAuthConfig,
+      );
+
+      // AUTODETECT entry should be skipped when createLlmApi returns null
+      expect(result).toHaveLength(0);
+    });
+  });
 });

extensions/cli/src/services/ModelService.ts

@@ -7,6 +7,65 @@ import { logger } from "../util/logger.js";
 import { BaseService, ServiceWithDependencies } from "./BaseService.js";
 import { AgentFileServiceState, ModelServiceState } from "./types.js";
 
+const AUTODETECT = "AUTODETECT";
+
+/**
+ * Expands AUTODETECT model entries into concrete models
+ * For each model with model === "AUTODETECT", calls llm.list() to get available models
+ * and replaces the AUTODETECT entry with one ModelConfig per available model.
+ * Handles errors gracefully by skipping failed expansions with a warning.
+ */
+export async function expandAutodetectModels(
+  models: ModelConfig[],
+  authConfig: AuthConfig,
+): Promise<ModelConfig[]> {
+  const expanded: ModelConfig[] = [];
+
+  for (const model of models) {
+    if (model.model !== AUTODETECT) {
+      expanded.push(model);
+      continue;
+    }
+
+    const llmApi = createLlmApi(model, authConfig);
+    if (!llmApi) {
+      logger.warn(
+        `Cannot create API client for AUTODETECT model with provider ${model.provider}, skipping`,
+      );
+      continue;
+    }
+
+    try {
+      const availableModels = await llmApi.list();
+      if (!availableModels || availableModels.length === 0) {
+        logger.warn(
+          `No models returned from ${model.provider} for AUTODETECT, skipping`,
+        );
+        continue;
+      }
+
+      for (const available of availableModels) {
+        if (available.id === AUTODETECT) continue;
+        expanded.push({
+          ...model,
+          model: available.id,
+          name: available.id,
+        });
+      }
+
+      logger.debug(
+        `Expanded AUTODETECT for ${model.provider}: ${availableModels.map((m) => m.id).join(", ")}`,
+      );
+    } catch (e) {
+      logger.warn(
+        `Error listing models for AUTODETECT expansion (${model.provider}): ${e}`,
+      );
+    }
+  }
+
+  return expanded;
+}
+
 /**
  * Service for managing LLM and model state
  * Depends on auth config and assistant config
@@ -49,8 +108,20 @@ export class ModelService
       assistantModelsCount: assistant?.models?.length || 0,
     });
 
-    this.assistant = assistant;
     this.authConfig = authConfig;
+
+    // Expand AUTODETECT model entries into concrete models
+    if (assistant.models?.some((m) => m?.model === AUTODETECT)) {
+      assistant = {
+        ...assistant,
+        models: await expandAutodetectModels(
+          assistant.models.filter((m): m is ModelConfig => !!m),
+          authConfig,
+        ),
+      };
+    }
+
+    this.assistant = assistant;
     this.availableModels = (assistant.models?.filter(
       (model) =>
         model && (model.roles?.includes("chat") || model.roles === undefined),