diff --git a/e2e/questdb b/e2e/questdb index 00de5bbb3..f2e1a8de1 160000 --- a/e2e/questdb +++ b/e2e/questdb @@ -1 +1 @@ -Subproject commit 00de5bbb3aed7e23794e6d7ff0cd4089a8e1423b +Subproject commit f2e1a8de1ab1201f57ca144b71c36ea0f1358736 diff --git a/e2e/tests/console/aiAssistant.spec.js b/e2e/tests/console/aiAssistant.spec.js index e362d6e54..5db93f04f 100644 --- a/e2e/tests/console/aiAssistant.spec.js +++ b/e2e/tests/console/aiAssistant.spec.js @@ -2,6 +2,7 @@ const { PROVIDERS, + interceptAIChatRequest, CUSTOM_PROVIDER_DEFAULTS, getOpenAIConfiguredSettings, getAnthropicConfiguredSettings, @@ -46,112 +47,6 @@ function interceptAIRequestWithResponse( }).as(aliasName) } -/** - * Intercepts AI chat requests with a default test response. - * - * @param {"anthropic" | "openai"} provider - The AI provider to intercept - * @param {string} [alias] - Optional custom alias for the intercept - * @param {number} [delay=0] - Delay in milliseconds - * @param {Object} [options] - Options - * @param {boolean} [options.streaming=true] - Whether to use streaming response - */ -function interceptAIChatRequest( - provider, - alias, - delay = 200, - options = { streaming: true }, -) { - const aliasName = alias || `${provider}ChatRequest` - const endpoint = PROVIDERS[provider].endpoint - const { streaming = true } = options - - const responseData = createFinalResponseData( - provider, - "Test response explanation", - ) - - cy.intercept("POST", endpoint, (req) => { - if (isTitleRequest(provider, req.body)) { - req.reply(createChatTitleResponse(provider, "Test Chat")) - return - } - req.alias = aliasName - req.reply(createResponse(provider, responseData, { streaming, delay })) - }) -} - -/** - * Intercepts AI provider token validation requests. - * - * @param {"anthropic" | "openai"} provider - The AI provider to intercept - * @param {boolean} success - If true, returns 200 success response; if false, returns 401 error - */ -function interceptTokenValidation(provider, success) { - const endpoint = PROVIDERS[provider].endpoint - - if (provider === "openai") { - if (success) { - cy.intercept("POST", endpoint, { - statusCode: 200, - delay: 200, - body: { - id: "resp_mock_test", - object: "response", - created_at: Date.now(), - status: "completed", - output: [], - }, - }).as("openaiValidation") - } else { - cy.intercept("POST", endpoint, { - statusCode: 401, - delay: 200, - body: { - error: { - message: - "Incorrect API key provided: ***. You can find your API key at https://platform.openai.com/account/api-keys.", - type: "invalid_request_error", - param: null, - code: "invalid_api_key", - }, - }, - }).as("openaiValidation") - } - } else if (provider === "anthropic") { - if (success) { - cy.intercept("POST", endpoint, { - statusCode: 200, - delay: 200, - body: { - id: "msg_mock_test", - type: "message", - role: "assistant", - content: [], - model: "claude-sonnet-4-5", - stop_reason: "end_turn", - usage: { - input_tokens: 10, - output_tokens: 5, - }, - }, - }).as("anthropicValidation") - } else { - cy.intercept("POST", endpoint, { - statusCode: 401, - delay: 200, - body: { - type: "error", - error: { - type: "authentication_error", - message: "invalid x-api-key", - }, - request_id: "req_mock_test", - }, - }).as("anthropicValidation") - } - } -} - describe("ai assistant", () => { beforeEach(() => { cy.intercept("POST", PROVIDERS.openai.endpoint, (req) => { @@ -165,405 +60,14 @@ describe("ai assistant", () => { `Unhandled Anthropic request detected! Request body: ${JSON.stringify(req.body).slice(0, 200)}...`, ) }).as("unhandledAnthropic") - }) - - describe("onboarding and settings", () => { - beforeEach(() => { - cy.loadConsoleWithAuth() - }) - - it("should display ai assistant promo", () => { - // When - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - - // Then - cy.getByDataHook("ai-promo-modal").should("be.visible") - - // When - cy.getByDataHook("ai-promo-close").should("be.visible").click() - - // Then - cy.getByDataHook("ai-promo-modal").should("not.exist") - - // When - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - cy.getByDataHook("ai-promo-continue").should("be.visible").click() - - // Then - cy.getByDataHook("ai-settings-modal-step-one").should("be.visible") - }) - - it("should handle invalid api key", () => { - // When - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - cy.getByDataHook("ai-promo-continue").should("be.visible").click() - - // Then - cy.getByDataHook("ai-settings-modal-step-one").should("be.visible") - // API key input is hidden until a provider is selected - cy.getByDataHook("ai-settings-api-key").should("not.exist") - - // When - select Anthropic - cy.getByDataHook("ai-settings-provider-anthropic").click() - - // Then - API key input appears - cy.getByDataHook("ai-settings-api-key") - .should("be.visible") - .should("have.attr", "placeholder", "Enter Anthropic API key") - - // When - switch to OpenAI - cy.getByDataHook("ai-settings-provider-openai").click() - - // Then - cy.getByDataHook("ai-settings-api-key") - .should("be.visible") - .should("have.attr", "placeholder", "Enter OpenAI API key") - ;["anthropic", "openai"].forEach((provider) => { - // Given - interceptTokenValidation(provider, false) - - // When - cy.getByDataHook(`ai-settings-provider-${provider}`).click() - - // Then - cy.getByDataHook("ai-settings-api-key") - .should("be.visible") - .should( - "have.attr", - "placeholder", - `Enter ${provider === "anthropic" ? "Anthropic" : "OpenAI"} API key`, - ) - .should("be.empty") - - // When - cy.getByDataHook("ai-settings-api-key").type("invalid-api-key") - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - cy.getByDataHook("multi-step-modal-next-button") - .should("be.disabled") - .should("contain", "Validating...") - - // When - cy.wait(`@${provider}Validation`) - - // Then - cy.getByDataHook("ai-settings-api-key-error").should("be.visible") - }) - }) - - it("should handle valid api key", () => { - // Given - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - cy.getByDataHook("ai-promo-continue").should("be.visible").click() - ;["anthropic", "openai"].forEach((provider) => { - // Given - interceptTokenValidation(provider, true) - - // When - cy.getByDataHook(`ai-settings-provider-${provider}`).click() - - // When - cy.getByDataHook("ai-settings-api-key").type("valid-api-key") - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") - cy.getByDataHook("multi-step-modal-cancel-button").click() - }) - }) - - it("should show ai buttons after setup is completed", () => { - // Given - interceptTokenValidation("openai", true) - - // When - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - cy.getByDataHook("ai-promo-continue").should("be.visible").click() - cy.getByDataHook("ai-settings-provider-openai").click() - cy.getByDataHook("ai-settings-api-key").type("valid-api-key") - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") - - // When - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - cy.getByDataHook("ai-assistant-settings-button").should( - "contain", - "AI Settings", - ) - cy.getByDataHook("ai-chat-button").should("be.visible") - cy.getByDataHook("ai-settings-model-dropdown").should("be.visible") - - // When / Then — selecting a model closes the dropdown (handleModelSelect - // calls setDropdownActive(false)), so re-open it for each model and - // re-query the item just before clicking; otherwise the list detaches the - // node as it settles/closes and cy.click() hits a stale element. - ;[0, 1].forEach((index) => { - cy.getByDataHook("ai-settings-model-dropdown").click() - cy.getByDataHook("ai-settings-model-item").should("be.visible") - - cy.getByDataHook("ai-settings-model-item") - .eq(index) - .find("[data-hook='ai-settings-model-item-label']") - .invoke("text") - .then((text) => { - const label = text.trim() - cy.getByDataHook("ai-settings-model-item").eq(index).click() - cy.getByDataHook("ai-settings-model-dropdown").should( - "contain", - label, - ) - }) - }) - - // When - cy.typeQuery("SELECT 1;") - - // Then - cy.getAIIconInLine(1).should("be.visible") - - // When - cy.getByDataHook("ai-assistant-settings-button").click() - - // Then - cy.getByDataHook("ai-settings-validated-badge") - .should("be.visible") - .should("contain", "Validated") - cy.getByDataHook("ai-settings-provider-openai") - .getByDataHook("ai-settings-provider-status") - .should("be.visible") - .should("contain", "Enabled") - - cy.getByDataHook("ai-settings-provider-anthropic") - .getByDataHook("ai-settings-provider-status") - .should("be.visible") - .should("contain", "Inactive") - - // When - cy.getByDataHook("ai-settings-remove-provider").scrollIntoView() - cy.getByDataHook("ai-settings-remove-provider") - .should("be.visible") - .click() - - // Then - cy.getByDataHook("ai-settings-validated-badge").should("not.exist") - cy.getByDataHook("ai-settings-provider-openai") - .getByDataHook("ai-settings-provider-status") - .should("be.visible") - .should("contain", "Inactive") - - // When - cy.getByDataHook("ai-settings-save").click() - - // Then - cy.getByDataHook("ai-settings-model-dropdown").should("not.exist") - cy.getByDataHook("ai-chat-button").should("not.exist") - cy.getByDataHook("ai-assistant-settings-button").should( - "contain", - "Configure", - ) - }) - - it("should not provide schema tools when schema access is disabled", () => { - const schemaTools = ["get_tables", "get_table_schema"] - - // Given - interceptTokenValidation("openai", true) - - // When - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - cy.getByDataHook("ai-promo-continue").should("be.visible").click() - cy.getByDataHook("ai-settings-provider-openai").click() - cy.getByDataHook("ai-settings-api-key").type("valid-api-key") - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") - - // When - drop permissions to None so schema tools are excluded. - cy.getByDataHook("permissions-trigger").click() - cy.getByDataHook("permission-level-none").click() - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - AI chat should be available - cy.get(".toast-success-container").should("be.visible").click() - cy.getByDataHook("ai-chat-button").should("be.visible") - - // When - Open chat and send a message - interceptAIChatRequest("openai", "chatWithoutSchema") - cy.getByDataHook("ai-chat-button").click() - cy.getByDataHook("ai-chat-window").should("be.visible") - cy.getByDataHook("chat-input-textarea").type("Hello, test message") - cy.getByDataHook("chat-send-button").click() - - // Then - Verify request does NOT contain schema tools - cy.wait("@chatWithoutSchema").then((interception) => { - const tools = interception.request.body.tools || [] - const toolNames = tools.map((t) => t.name || t.function?.name) - schemaTools.forEach((schemaTool) => { - expect(toolNames).to.not.include(schemaTool) - }) - }) - - // When - Open settings modal and re-enable schema access - cy.getByDataHook("ai-assistant-settings-button").click() - cy.getByDataHook("permissions-trigger").click() - cy.getByDataHook("permission-level-schema").click() - cy.getByDataHook("ai-settings-save").click() - cy.get(".toast-success-container").should("be.visible").click() - - // When - Send another message - interceptAIChatRequest("openai", "chatWithSchema") - cy.getByDataHook("chat-input-textarea").type("Another test message") - cy.getByDataHook("chat-send-button").click() - - // Then - Verify request DOES contain schema tools - cy.wait("@chatWithSchema").then((interception) => { - const tools = interception.request.body.tools || [] - const toolNames = tools.map((t) => t.name || t.function?.name) - schemaTools.forEach((schemaTool) => { - expect(toolNames).to.include(schemaTool) - }) - }) - }) - - it("should work with multiple providers", () => { - const openaiEnabledModels = [] - const anthropicEnabledModels = [] - - // Given - Set up OpenAI provider first - interceptTokenValidation("openai", true) - - // When - Complete setup with OpenAI - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - cy.getByDataHook("ai-promo-continue").should("be.visible").click() - cy.getByDataHook("ai-settings-provider-openai").click() - cy.getByDataHook("ai-settings-api-key").type("valid-openai-key") - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - Should be on step two - cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") - - // When - Store enabled model labels for OpenAI - cy.get('[data-model-enabled="true"]').each(($modelRow) => { - openaiEnabledModels.push($modelRow.attr("data-model")) - }) - cy.getByDataHook("multi-step-modal-next-button").click() + cy.intercept("GET", "https://api.openai.com/v1/models*", () => { + throw new Error("Unhandled OpenAI model listing request detected!") + }).as("unhandledOpenAIModels") - // Then - Verify model dropdown shows exactly the enabled OpenAI models - cy.get(".toast-success-container").should("be.visible").click() - cy.getByDataHook("ai-settings-model-dropdown").click() - cy.then(() => { - cy.getByDataHook("ai-settings-model-item").should( - "have.length", - openaiEnabledModels.length, - ) - openaiEnabledModels.forEach((modelLabel) => { - cy.getByDataHook("ai-settings-model-item").contains(modelLabel) - }) - }) - cy.get("body").type("{esc}") // close dropdown - - // When - Open settings and configure Anthropic provider - interceptTokenValidation("anthropic", true) - cy.getByDataHook("ai-assistant-settings-button").click() - - // Then - OpenAI should show Enabled, Anthropic should show Inactive - cy.getByDataHook("ai-settings-provider-openai") - .getByDataHook("ai-settings-provider-status") - .should("contain", "Enabled") - cy.getByDataHook("ai-settings-provider-anthropic") - .getByDataHook("ai-settings-provider-status") - .should("contain", "Inactive") - - // When - Configure Anthropic - cy.getByDataHook("ai-settings-provider-anthropic").click() - cy.getByDataHook("ai-settings-api-key").type("valid-anthropic-key") - cy.getByDataHook("ai-settings-test-api").click() - - // Then - Should show validating and then validated - cy.wait("@anthropicValidation") - - // Then - Anthropic should no longer show Inactive - cy.getByDataHook("ai-settings-provider-anthropic") - .getByDataHook("ai-settings-provider-status") - .should("not.contain", "Inactive") - - // When - Store enabled model labels for Anthropic - cy.get('[data-enabled="true"]').each(($modelRow) => { - anthropicEnabledModels.push($modelRow.attr("data-model")) - }) - - // When - Save settings - cy.getByDataHook("ai-settings-save").click() - cy.get(".toast-success-container").should("be.visible").click() - - // Then - Model dropdown should contain models from both providers - cy.getByDataHook("ai-settings-model-dropdown").click() - cy.then(() => { - const allEnabledModels = [ - ...openaiEnabledModels, - ...anthropicEnabledModels, - ] - cy.getByDataHook("ai-settings-model-item").should( - "have.length", - allEnabledModels.length, - ) - allEnabledModels.forEach((modelLabel) => { - cy.getByDataHook("ai-settings-model-item").contains(modelLabel) - }) - }) - - // When - Select first OpenAI model and open chat - cy.then(() => { - cy.getByDataHook("ai-settings-model-item") - .contains(openaiEnabledModels[0]) - .click() - }) - interceptAIChatRequest("openai", "openaiChat") - cy.getByDataHook("ai-chat-button").click() - cy.getByDataHook("ai-chat-window").should("be.visible") - cy.getByDataHook("chat-input-textarea").type("Test message for OpenAI") - cy.getByDataHook("chat-send-button").click() - - // Then - Should intercept OpenAI request - cy.wait("@openaiChat") - - // When - Select first Anthropic model from dropdown - cy.getByDataHook("ai-settings-model-dropdown").click() - cy.then(() => { - cy.getByDataHook("ai-settings-model-item") - .contains(anthropicEnabledModels[0]) - .click() - }) - - // When - Send another message - interceptAIChatRequest("anthropic", "anthropicChat") - cy.getByDataHook("chat-input-textarea").type("Test message for Anthropic") - cy.getByDataHook("chat-send-button").click() - - // Then - Should intercept Anthropic request - cy.wait("@anthropicChat") - }) + cy.intercept("GET", "https://api.anthropic.com/v1/models*", () => { + throw new Error("Unhandled Anthropic model listing request detected!") + }).as("unhandledAnthropicModels") }) describe("ai chat window ergonomics", () => { @@ -3499,15 +3003,32 @@ describe("custom providers", () => { cy.contains("codellama").should("be.visible") cy.get("body").type("{esc}") // close dropdown + cy.intercept("GET", "http://localhost:11434/v1/models*", { + statusCode: 200, + body: { + object: "list", + data: [ + { id: "llama3", object: "model" }, + { id: "mistral", object: "model" }, + { id: "codellama", object: "model" }, + ], + }, + }).as("ollamaModels") + cy.getByDataHook("ai-assistant-settings-button").click() cy.getByDataHook("ai-settings-provider-ollama").should("be.visible").click() + // Models render as read-only rows; changes go through Manage models cy.get("[data-model='llama3']").should("exist") cy.get("[data-model='mistral']").should("exist") cy.get("[data-model='codellama']").should("exist") - cy.get("[data-model='mistral']").find("button[role='switch']").click() - cy.get("[data-model='mistral'][data-enabled='false']").should("exist") + cy.getByDataHook("ai-settings-manage-models").click() + cy.wait("@ollamaModels") + cy.getByDataHook("custom-provider-model-row").contains("mistral").click() + cy.getByDataHook("manage-models-save").click() + + cy.get("[data-model='mistral']").should("not.exist") cy.getByDataHook("ai-settings-save").click() cy.getByDataHook("ai-settings-model-dropdown").should("be.visible").click() @@ -3518,10 +3039,14 @@ describe("custom providers", () => { cy.getByDataHook("ai-assistant-settings-button").click() cy.getByDataHook("ai-settings-provider-ollama").click() - cy.get("[data-model='mistral'][data-enabled='false']").should("exist") + cy.get("[data-model='mistral']").should("not.exist") + + cy.getByDataHook("ai-settings-manage-models").click() + cy.wait("@ollamaModels") + cy.getByDataHook("custom-provider-model-row").contains("mistral").click() + cy.getByDataHook("manage-models-save").click() - cy.get("[data-model='mistral']").find("button[role='switch']").click() - cy.get("[data-model='mistral'][data-enabled='true']").should("exist") + cy.get("[data-model='mistral']").should("exist") cy.getByDataHook("ai-settings-save").click() cy.getByDataHook("ai-settings-model-dropdown").should("be.visible").click() @@ -3859,7 +3384,7 @@ describe("custom providers", () => { cy.getByDataHook("ai-settings-model-dropdown").click() cy.getByDataHook("ai-settings-model-item-label") - .contains("GPT-5 mini") + .contains("GPT 5 Mini") .click() cy.getByDataHook("chat-window-new").click() @@ -4158,7 +3683,7 @@ describe("custom providers", () => { cy.get("body").type("{esc}") // close dropdown }) - it("should auto-enable new models from manage models and preserve unsaved toggle state", () => { + it("should add and remove models through manage models in manual mode", () => { const providerId = "test-provider" cy.loadConsoleWithAuth( @@ -4175,14 +3700,10 @@ describe("custom providers", () => { .should("be.visible") .click() - // All 3 models should be enabled - cy.get("[data-model='model-a'][data-enabled='true']").should("exist") - cy.get("[data-model='model-b'][data-enabled='true']").should("exist") - cy.get("[data-model='model-c'][data-enabled='true']").should("exist") - - // Disable model-b toggle (unsaved state) - cy.get("[data-model='model-b']").find("button[role='switch']").click() - cy.get("[data-model='model-b'][data-enabled='false']").should("exist") + // All 3 models render as read-only rows + cy.get("[data-model='model-a']").should("exist") + cy.get("[data-model='model-b']").should("exist") + cy.get("[data-model='model-c']").should("exist") // Intercept model fetch → fail to get manual mode cy.intercept("GET", "**/models*", { @@ -4214,10 +3735,10 @@ describe("custom providers", () => { cy.getByDataHook("manage-models-save").click() // Back in SettingsModal: model-b gone, model-d auto-enabled - cy.get("[data-model='model-a'][data-enabled='true']").should("exist") + cy.get("[data-model='model-a']").should("exist") cy.get("[data-model='model-b']").should("not.exist") - cy.get("[data-model='model-c'][data-enabled='true']").should("exist") - cy.get("[data-model='model-d'][data-enabled='true']").should("exist") + cy.get("[data-model='model-c']").should("exist") + cy.get("[data-model='model-d']").should("exist") // Save settings cy.getByDataHook("ai-settings-save").click() @@ -4234,10 +3755,6 @@ describe("custom providers", () => { it("should handle no-API-key custom provider: models visible, no validated badge, schema toggle enabled, and allow adding an API key", () => { const providerId = "ollama" - const customEndpoint = getCustomProviderEndpoint( - CUSTOM_PROVIDER_DEFAULTS.baseURL, - "openai-chat-completions", - ) cy.loadConsoleWithAuth( false, @@ -4265,7 +3782,7 @@ describe("custom providers", () => { "This provider does not have an API key", ) - // Model list visible with both models + // Model list visible with both models as read-only rows cy.get("[data-model='llama3']").should("exist") cy.get("[data-model='mistral']").should("exist") @@ -4275,11 +3792,7 @@ describe("custom providers", () => { // Manage models button visible cy.getByDataHook("ai-settings-manage-models").should("be.visible") - // Toggle mistral off - cy.get("[data-model='mistral']").find("button[role='switch']").click() - cy.get("[data-model='mistral'][data-enabled='false']").should("exist") - - // Built-in provider should NOT have manage models button + // Built-in provider should NOT have manage models button before validation cy.getByDataHook("ai-settings-provider-openai").click() cy.getByDataHook("ai-settings-manage-models").should("not.exist") @@ -4292,21 +3805,16 @@ describe("custom providers", () => { cy.getByDataHook("ai-settings-edit-api-key").click() cy.getByDataHook("ai-settings-api-key").type("sk-custom-key-123") - // Intercept validation request to custom endpoint - cy.intercept("POST", customEndpoint, { + // Validation runs through the provider's model listing + cy.intercept("GET", "http://localhost:11434/v1/models*", { statusCode: 200, delay: 200, body: { - id: "chatcmpl-mock", - object: "chat.completion", - choices: [ - { - index: 0, - message: { role: "assistant", content: "" }, - finish_reason: "stop", - }, + object: "list", + data: [ + { id: "llama3", object: "model" }, + { id: "mistral", object: "model" }, ], - usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, }, }).as("customValidation") @@ -4317,19 +3825,20 @@ describe("custom providers", () => { // Validated badge should now appear cy.getByDataHook("ai-settings-validated-badge").should("be.visible") - // Models still visible, mistral toggle preserved - cy.get("[data-model='llama3'][data-enabled='true']").should("exist") - cy.get("[data-model='mistral'][data-enabled='false']").should("exist") + // Models still visible + cy.get("[data-model='llama3']").should("exist") + cy.get("[data-model='mistral']").should("exist") // Part C: Save and verify cy.getByDataHook("ai-settings-save").click() cy.get(".toast-success-container").should("be.visible").click() - // Dropdown should show only llama3 (mistral was disabled) + // Dropdown should show both models cy.getByDataHook("ai-settings-model-dropdown").should("be.visible").click() - cy.getByDataHook("ai-settings-model-item").should("have.length", 1) + cy.getByDataHook("ai-settings-model-item").should("have.length", 2) cy.contains("llama3").should("be.visible") + cy.contains("mistral").should("be.visible") cy.get("body").type("{esc}") // close dropdown }) }) diff --git a/e2e/tests/console/aiProviderSetup.spec.js b/e2e/tests/console/aiProviderSetup.spec.js new file mode 100644 index 000000000..57ae16e52 --- /dev/null +++ b/e2e/tests/console/aiProviderSetup.spec.js @@ -0,0 +1,1117 @@ +/// + +const { + PROVIDERS, + interceptAIChatRequest, + getAnthropicConfiguredSettings, + createResponse, + createFinalResponseData, + createChatTitleResponse, + isTitleRequest, +} = require("../../utils/aiAssistant") + +const OPENAI_MODELS_URL = "https://api.openai.com/v1/models*" +const ANTHROPIC_MODELS_URL = "https://api.anthropic.com/v1/models*" + +const OPENAI_LISTING = { + object: "list", + data: [ + { id: "gpt-5.4", object: "model", created: 1772000000 }, + { id: "gpt-5-mini", object: "model", created: 1754500000 }, + { id: "gpt-5", object: "model", created: 1754400000 }, + { id: "gpt-5-nano", object: "model", created: 1754300000 }, + { id: "gpt-5-2025-08-06", object: "model", created: 1754400000 }, + { id: "whisper-1", object: "model", created: 1677532384 }, + ], +} + +const ANTHROPIC_LISTING = { + data: [ + { + type: "model", + id: "claude-opus-4-5", + display_name: "Claude Opus 4.5", + created_at: "2025-11-01T00:00:00Z", + }, + { + type: "model", + id: "claude-sonnet-4-5", + display_name: "Claude Sonnet 4.5", + created_at: "2025-09-29T00:00:00Z", + }, + { + type: "model", + id: "claude-haiku-4-5", + display_name: "Claude Haiku 4.5", + created_at: "2025-10-01T00:00:00Z", + }, + ], + has_more: false, + first_id: "claude-opus-4-5", + last_id: "claude-haiku-4-5", +} + +function interceptOpenAIListing(options = {}) { + cy.intercept("GET", OPENAI_MODELS_URL, { + statusCode: 200, + delay: options.delay ?? 200, + body: OPENAI_LISTING, + }).as("openaiListing") +} + +function interceptAnthropicListing(options = {}) { + cy.intercept("GET", ANTHROPIC_MODELS_URL, { + statusCode: 200, + delay: options.delay ?? 200, + body: ANTHROPIC_LISTING, + }).as("anthropicListing") +} + +function readAiSettings(win) { + return JSON.parse(win.localStorage.getItem("ai.assistant.settings")) +} + +/** + * Intercepts AI provider model listing requests. + * Validation now runs through GET /v1/models, so a mocked listing both + * validates the key and feeds the model picker. + * + * The OpenAI listing carries noise (whisper-1) that the picker must filter, + * and `created` timestamps that drive newest-first ordering. + * + * @param {"anthropic" | "openai"} provider - The AI provider to intercept + * @param {boolean} success - If true, returns 200 with a listing; if false, returns 401 + */ +function interceptTokenValidation(provider, success) { + if (provider === "openai") { + if (success) { + cy.intercept("GET", "https://api.openai.com/v1/models*", { + statusCode: 200, + delay: 200, + body: { + object: "list", + data: [ + { id: "gpt-5.4", object: "model", created: 1772000000 }, + { id: "gpt-5-mini", object: "model", created: 1754500000 }, + { id: "gpt-5", object: "model", created: 1754400000 }, + { id: "gpt-5-nano", object: "model", created: 1754300000 }, + { id: "whisper-1", object: "model", created: 1677532384 }, + ], + }, + }).as("openaiValidation") + } else { + cy.intercept("GET", "https://api.openai.com/v1/models*", { + statusCode: 401, + delay: 200, + body: { + error: { + message: + "Incorrect API key provided: ***. You can find your API key at https://platform.openai.com/account/api-keys.", + type: "invalid_request_error", + param: null, + code: "invalid_api_key", + }, + }, + }).as("openaiValidation") + } + } else if (provider === "anthropic") { + if (success) { + cy.intercept("GET", "https://api.anthropic.com/v1/models*", { + statusCode: 200, + delay: 200, + body: { + data: [ + { + type: "model", + id: "claude-opus-4-5", + display_name: "Claude Opus 4.5", + created_at: "2025-11-01T00:00:00Z", + }, + { + type: "model", + id: "claude-sonnet-4-5", + display_name: "Claude Sonnet 4.5", + created_at: "2025-09-29T00:00:00Z", + }, + { + type: "model", + id: "claude-haiku-4-5", + display_name: "Claude Haiku 4.5", + created_at: "2025-10-01T00:00:00Z", + }, + ], + has_more: false, + first_id: "claude-opus-4-5", + last_id: "claude-haiku-4-5", + }, + }).as("anthropicValidation") + } else { + cy.intercept("GET", "https://api.anthropic.com/v1/models*", { + statusCode: 401, + delay: 200, + body: { + type: "error", + error: { + type: "authentication_error", + message: "invalid x-api-key", + }, + request_id: "req_mock_test", + }, + }).as("anthropicValidation") + } + } +} + +describe("ai provider setup flows", () => { + beforeEach(() => { + cy.intercept("POST", PROVIDERS.openai.endpoint, (req) => { + throw new Error( + `Unhandled OpenAI request detected! Request body: ${JSON.stringify(req.body).slice(0, 200)}...`, + ) + }).as("unhandledOpenAI") + + cy.intercept("POST", PROVIDERS.anthropic.endpoint, (req) => { + throw new Error( + `Unhandled Anthropic request detected! Request body: ${JSON.stringify(req.body).slice(0, 200)}...`, + ) + }).as("unhandledAnthropic") + + cy.intercept("GET", "https://api.openai.com/v1/models*", () => { + throw new Error("Unhandled OpenAI model listing request detected!") + }).as("unhandledOpenAIModels") + + cy.intercept("GET", "https://api.anthropic.com/v1/models*", () => { + throw new Error("Unhandled Anthropic model listing request detected!") + }).as("unhandledAnthropicModels") + }) + + it("onboards a first-run OpenAI user from key to a reasoning chat and its fallback", () => { + // Given a fresh console with intercepted OpenAI endpoints + cy.loadConsoleWithAuth() + cy.intercept("GET", OPENAI_MODELS_URL, { + statusCode: 401, + delay: 200, + body: { + error: { + message: "Incorrect API key provided", + type: "invalid_request_error", + param: null, + code: "invalid_api_key", + }, + }, + }).as("openaiListing") + + // When the wizard opens and an invalid key is validated + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-promo-continue").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("invalid-key") + cy.getByDataHook("multi-step-modal-next-button").click() + cy.wait("@openaiListing") + + // Then the field shows an inline error and stays on step one + cy.getByDataHook("ai-settings-api-key-error").should( + "contain", + "Invalid API key", + ) + cy.getByDataHook("ai-settings-modal-step-one").should("be.visible") + + // When the key is corrected and validated + interceptOpenAIListing() + cy.getByDataHook("ai-settings-api-key").clear().type("valid-key") + cy.getByDataHook("multi-step-modal-next-button").click() + cy.wait("@openaiListing") + cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") + + // Then activating with no model shows the footer error bar + cy.getByDataHook("multi-step-modal-next-button").click() + cy.getByDataHook("multi-step-modal-error").should( + "contain", + "Please enable at least one model", + ) + + // When a listed model and a manual model are enabled + cy.getByDataHook("configure-models-model-row").contains("gpt-5.4").click() + cy.getByDataHook("configure-models-manual-model-input").type( + "my-proxy-model", + ) + cy.getByDataHook("configure-models-add-model-button").click() + + // Then the manual model shows as a chip and the error bar is gone on retry + cy.getByDataHook("configure-models-model-chip").should( + "contain", + "my-proxy-model", + ) + + // When reasoning is set to High and the assistant is activated + cy.getByDataHook("reasoning-trigger").click() + cy.getByDataHook("reasoning-level-high").click() + cy.getByDataHook("multi-step-modal-next-button").click() + cy.get(".toast-success-container").should("be.visible").click() + + // Then the persisted settings carry the whole configuration + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.selectedModel).to.equal("openai:gpt-5.4") + expect(settings.providers.openai.enabledModels).to.deep.equal([ + "openai:gpt-5.4", + "openai:my-proxy-model", + ]) + expect(settings.providers.openai.reasoningEffort).to.equal("high") + expect(settings.providers.openai.utilityModel).to.equal( + "openai:gpt-5-nano", + ) + expect(settings.providers.openai.modelLabels).to.deep.equal({ + "gpt-5.4": "GPT 5.4", + "my-proxy-model": "my-proxy-model", + }) + }) + + // When a chat message is sent + cy.intercept("POST", PROVIDERS.openai.endpoint, (req) => { + if (isTitleRequest("openai", req.body)) { + expect(req.body.model).to.equal("gpt-5-nano") + req.reply(createChatTitleResponse("openai", "Test Chat")) + return + } + expect(req.body.model).to.equal("gpt-5.4") + req.alias = "reasoningChat" + req.reply( + createResponse( + "openai", + createFinalResponseData("openai", "First answer"), + { streaming: req.body.stream === true }, + ), + ) + }) + cy.getByDataHook("ai-chat-button").click() + cy.getByDataHook("chat-input-textarea").type("hello") + cy.getByDataHook("chat-send-button").click() + + // Then the request leaves the browser with high reasoning effort + cy.wait("@reasoningChat") + .its("request.body.reasoning") + .should("deep.equal", { effort: "high", summary: "auto" }) + + // When the model rejects the reasoning parameter on the next message + cy.intercept("POST", PROVIDERS.openai.endpoint, (req) => { + if (isTitleRequest("openai", req.body)) { + req.reply(createChatTitleResponse("openai", "Test Chat")) + return + } + if (req.body.reasoning) { + req.alias = "rejectedChat" + req.reply({ + statusCode: 400, + body: { + error: { + message: + "Unsupported parameter: 'reasoning.effort' is not supported with this model.", + type: "invalid_request_error", + param: "reasoning.effort", + code: "unsupported_parameter", + }, + }, + }) + return + } + req.alias = "strippedRetry" + req.reply( + createResponse( + "openai", + createFinalResponseData("openai", "Fallback answer"), + { streaming: req.body.stream === true }, + ), + ) + }) + cy.getByDataHook("chat-input-textarea").type("again") + cy.getByDataHook("chat-send-button").click() + + // Then the rejected request is retried without reasoning + cy.wait("@rejectedChat") + cy.wait("@strippedRetry").then((interception) => { + expect(interception.request.body).to.not.have.property("reasoning") + }) + + // And the downgrade is surfaced and persisted as Default + cy.get(".toast-info-container").should( + "contain", + "Reasoning preference changed to Default", + ) + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.providers.openai.reasoningEffort).to.equal("default") + }) + }) + + it("keeps the provider identity of identical built-in model ids", () => { + cy.loadConsoleWithAuth(false, { + "ai.assistant.settings": JSON.stringify({ + modelValueFormat: 2, + selectedModel: "anthropic:shared-model", + providers: { + anthropic: { + apiKey: "test-anthropic-key", + enabledModels: ["anthropic:shared-model"], + grantSchemaAccess: false, + }, + openai: { + apiKey: "test-openai-key", + enabledModels: ["openai:shared-model"], + grantSchemaAccess: false, + }, + }, + }), + }) + + // Both rows keep the raw model name as their visible label. + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.getByDataHook("ai-settings-model-item") + .should("have.length", 2) + .find("[data-hook='ai-settings-model-item-label']") + .each(($label) => { + expect($label.text()).to.equal("Shared Model") + }) + + // Selecting the OpenAI row persists its provider-qualified identity. + cy.getByDataHook("ai-settings-model-item").eq(1).click() + cy.window().then((win) => { + expect(readAiSettings(win).selectedModel).to.equal("openai:shared-model") + }) + + // Execution uses that identity to select the OpenAI provider. + interceptAIChatRequest("openai", "overlappingOpenAIModel") + cy.getByDataHook("ai-chat-button").click() + cy.getByDataHook("chat-input-textarea").type("hello") + cy.getByDataHook("chat-send-button").click() + cy.wait("@overlappingOpenAIModel") + .its("request.body.model") + .should("equal", "shared-model") + }) + + it("manages the OpenAI provider lifecycle from the settings modal", () => { + // Given a console already configured with Anthropic + cy.loadConsoleWithAuth(false, getAnthropicConfiguredSettings()) + interceptOpenAIListing() + + // When an OpenAI key validates + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("key-one") + cy.getByDataHook("ai-settings-test-api").click() + cy.wait("@openaiListing") + + // Then the picker auto-opens from the validation fetch, with no extra request + cy.getByDataHook("manage-models-model-row").should("have.length", 5) + cy.get("@openaiListing.all").should("have.length", 1) + + // When the picker is cancelled with nothing selected + cy.getByDataHook("manage-models-cancel").click() + + // Then the never-configured provider is dropped back to unvalidated + cy.getByDataHook("ai-settings-test-api").should("be.visible") + cy.getByDataHook("ai-settings-validated-badge").should("not.exist") + + // When validation runs again and Select All is used + cy.getByDataHook("ai-settings-test-api").click() + cy.wait("@openaiListing") + cy.getByDataHook("manage-models-model-row").should("have.length", 5) + cy.getByDataHook("manage-models-select-all").click() + cy.getByDataHook("manage-models-save").click() + cy.getByDataHook("manage-models-save").should("not.exist") + + // Then every listed chat model, including dated snapshots, persists + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.providers.openai.enabledModels).to.deep.equal([ + "openai:gpt-5.4", + "openai:gpt-5-mini", + "openai:gpt-5", + "openai:gpt-5-2025-08-06", + "openai:gpt-5-nano", + ]) + }) + + // When the parent modal closes without Save Settings and the page reloads + cy.getByDataHook("ai-settings-cancel").click() + cy.reload() + cy.getEditor().should("be.visible") + + // Then the picks survive and an OpenAI model can be selected + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.getByDataHook("ai-settings-model-item").should("have.length", 7) + cy.getByDataHook("ai-settings-model-item").contains("GPT 5.4").click() + + // When the picker reopens manually it refetches a fresh listing + interceptOpenAIListing() + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("reasoning-trigger").click() + cy.getByDataHook("reasoning-level-high").click() + cy.getByDataHook("permissions-trigger").click() + cy.getByDataHook("permission-level-write").click() + cy.getByDataHook("ai-settings-manage-models").click() + cy.wait("@openaiListing") + + // And unticking the selected model keeps the selection on OpenAI + cy.getByDataHook("manage-models-model-row") + .contains("gpt-5.4") + .closest("label") + .find("input[type=checkbox]") + .click() + cy.getByDataHook("manage-models-save").click() + cy.getByDataHook("manage-models-save").should("not.exist") + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.selectedModel).to.equal("openai:gpt-5-mini") + expect(settings.providers.openai.grantSchemaAccess).to.equal(true) + expect(settings.providers.openai.read).to.equal(false) + expect(settings.providers.openai.write).to.equal(false) + expect(settings.providers.openai.reasoningEffort).to.equal(undefined) + }) + + // Then cancelling the parent discards its permission and reasoning drafts + cy.getByDataHook("ai-settings-cancel").click() + cy.reload() + cy.getEditor().should("be.visible") + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.providers.openai.grantSchemaAccess).to.equal(true) + expect(settings.providers.openai.read).to.equal(false) + expect(settings.providers.openai.write).to.equal(false) + expect(settings.providers.openai.reasoningEffort).to.equal(undefined) + }) + + // When the API key changes to a different one and validates + interceptOpenAIListing() + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-edit-api-key").click() + cy.getByDataHook("ai-settings-api-key").should("not.have.attr", "readonly") + cy.getByDataHook("ai-settings-api-key").clear().type("key-two") + cy.getByDataHook("ai-settings-test-api").click() + cy.wait("@openaiListing") + + // Then the old key's picks are cleared in the picker + cy.getByDataHook("manage-models-model-row").should("have.length", 5) + cy.getByDataHook("manage-models-model-row") + .find("input[type=checkbox]:checked") + .should("have.length", 0) + + // And cancelling reverts to the stored working configuration + cy.getByDataHook("manage-models-cancel").click() + cy.getByDataHook("ai-settings-validated-badge").should("be.visible") + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.providers.openai.enabledModels).to.deep.equal([ + "openai:gpt-5-mini", + "openai:gpt-5", + "openai:gpt-5-2025-08-06", + "openai:gpt-5-nano", + ]) + }) + }) + + it("keeps delisted and manual models as removable chips with exact row identity", () => { + // Given stored models: a listed alias, a delisted dated snapshot, a manual id + cy.loadConsoleWithAuth(false, { + "ai.assistant.settings": JSON.stringify({ + selectedModel: "gpt-5.4", + providers: { + openai: { + apiKey: "test-openai-key", + enabledModels: ["gpt-5.4", "gpt-5.4-2026-03-05", "my-proxy-model"], + grantSchemaAccess: false, + }, + }, + }), + }) + interceptOpenAIListing() + + // When Manage Models opens + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-manage-models").click() + cy.wait("@openaiListing") + + // Then the listed alias is checked and the other two are plain chips + cy.getByDataHook("manage-models-model-row") + .contains("gpt-5.4") + .closest("label") + .find("input[type=checkbox]") + .should("be.checked") + cy.getByDataHook("manage-models-model-chip").should("have.length", 2) + cy.getByDataHook("manage-models-model-chip").contains("gpt-5.4-2026-03-05") + cy.getByDataHook("manage-models-model-chip").contains("my-proxy-model") + + // The plain alias and dated snapshot rows toggle independently + cy.getByDataHook("manage-models-model-row") + .contains(/^gpt-5$/) + .closest("label") + .find("input[type=checkbox]") + .click() + cy.getByDataHook("manage-models-model-row") + .contains("gpt-5-2025-08-06") + .closest("label") + .find("input[type=checkbox]") + .should("not.be.checked") + + // When the dated chip is removed and the picker saves + cy.getByDataHook("manage-models-model-chip") + .contains("gpt-5.4-2026-03-05") + .closest("[data-hook='manage-models-model-chip']") + .find("[data-hook='manage-models-remove-model']") + .click() + cy.getByDataHook("manage-models-save").click() + cy.getByDataHook("manage-models-save").should("not.exist") + + // Then only the removed id is gone — nothing was dropped silently + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.providers.openai.enabledModels).to.deep.equal([ + "openai:gpt-5.4", + "openai:my-proxy-model", + "openai:gpt-5", + ]) + }) + + // And the dropdown shows derived labels but never invents one for manual ids + cy.getByDataHook("ai-settings-cancel").click() + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.getByDataHook("ai-settings-model-item").contains("GPT 5.4") + cy.getByDataHook("ai-settings-model-item").contains("my-proxy-model") + cy.getByDataHook("ai-settings-model-item") + .contains("My-proxy-model") + .should("not.exist") + }) + + it("blocks model changes and identifies a rate-limited listing", () => { + // Given an existing OpenAI configuration whose model listing is rate limited + cy.loadConsoleWithAuth(false, { + "ai.assistant.settings": JSON.stringify({ + selectedModel: "gpt-5.4", + providers: { + openai: { + apiKey: "test-openai-key", + enabledModels: ["gpt-5.4"], + grantSchemaAccess: false, + }, + }, + }), + }) + cy.intercept("GET", OPENAI_MODELS_URL, { + statusCode: 429, + headers: { "retry-after": "0" }, + body: { + error: { + message: "Rate limit reached", + type: "rate_limit_error", + param: null, + code: "rate_limit_exceeded", + }, + }, + }).as("openaiListing") + + // When Manage Models attempts to refresh the provider listing + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-manage-models").click() + cy.wait("@openaiListing") + + // Then the failure is identified and no model changes can be made + cy.getByDataHook("manage-models-fetch-error").should( + "contain", + "rate or usage limit was reached", + ) + cy.getByDataHook("manage-models-model-row").should("not.exist") + cy.getByDataHook("manage-models-manual-model-input").should("not.exist") + cy.getByDataHook("manage-models-save").should("be.disabled") + + // And the saved configuration is untouched + cy.window().then((win) => { + expect(readAiSettings(win).providers.openai.enabledModels).to.deep.equal([ + "openai:gpt-5.4", + ]) + }) + }) + + it("survives interruptions: wizard escape, tab switch mid-validation, stale key edits", () => { + // Given a fresh console and a slow listing response + cy.loadConsoleWithAuth() + interceptOpenAIListing({ delay: 1500 }) + + // When the wizard validation is escaped mid-flight + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-promo-continue").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("valid-key") + cy.getByDataHook("multi-step-modal-next-button").click() + cy.get("body").type("{esc}") + cy.wait("@openaiListing") + + // Then reopening lands on a clean step one, not a dead-end step two + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-promo-continue").click() + cy.getByDataHook("ai-settings-modal-step-one").should("be.visible") + cy.getByDataHook("ai-settings-modal-step-two").should("not.exist") + + // When the wizard completes normally to reach the settings modal + interceptOpenAIListing() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("valid-key") + cy.getByDataHook("multi-step-modal-next-button").click() + cy.wait("@openaiListing") + cy.getByDataHook("configure-models-model-row").contains("gpt-5.4").click() + cy.getByDataHook("multi-step-modal-next-button").click() + cy.get(".toast-success-container").should("be.visible").click() + + // And an Anthropic validation starts while the tab switches to OpenAI + interceptAnthropicListing({ delay: 1500 }) + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-settings-provider-anthropic").click() + cy.getByDataHook("ai-settings-api-key").type("anthropic-key") + cy.getByDataHook("ai-settings-test-api").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.wait("@anthropicListing") + + // Then the tab switch aborted the validation: no picker, nothing validated + cy.getByDataHook("manage-models-model-row").should("not.exist") + cy.getByDataHook("ai-settings-provider-anthropic").click() + cy.getByDataHook("ai-settings-test-api") + .should("be.visible") + .should("not.be.disabled") + + // And validating without switching opens the Anthropic picker and saves there + interceptAnthropicListing() + cy.getByDataHook("ai-settings-test-api").click() + cy.wait("@anthropicListing") + cy.getByDataHook("manage-models-model-row").should("have.length", 3) + cy.get("[role=dialog]").should("contain", "Enable the Anthropic models") + cy.getByDataHook("manage-models-model-row") + .contains("Claude Haiku 4.5") + .click() + cy.getByDataHook("manage-models-save").click() + cy.getByDataHook("manage-models-save").should("not.exist") + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.providers.anthropic.enabledModels).to.deep.equal([ + "anthropic:claude-haiku-4-5", + ]) + expect(settings.providers.openai.enabledModels).to.deep.equal([ + "openai:gpt-5.4", + ]) + }) + + // When a validation response arrives for a key that was already edited + interceptOpenAIListing({ delay: 1500 }) + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-edit-api-key").click() + cy.getByDataHook("ai-settings-api-key").should("not.have.attr", "readonly") + cy.getByDataHook("ai-settings-api-key").clear().type("key-a") + cy.getByDataHook("ai-settings-test-api").click() + cy.getByDataHook("ai-settings-edit-api-key").click() + cy.getByDataHook("ai-settings-api-key").should("not.have.attr", "readonly") + cy.getByDataHook("ai-settings-api-key").clear() + cy.getByDataHook("ai-settings-api-key").type("key-b-changed") + cy.wait("@openaiListing") + + // Then the stale response is discarded: no badge, no picker, ready to validate + cy.getByDataHook("ai-settings-validated-badge").should("not.exist") + cy.getByDataHook("manage-models-model-row").should("not.exist") + cy.getByDataHook("ai-settings-test-api") + .should("be.visible") + .should("not.be.disabled") + }) + + describe("onboarding and settings", () => { + beforeEach(() => { + cy.loadConsoleWithAuth() + }) + + it("should display ai assistant promo", () => { + // When + cy.getByDataHook("ai-assistant-settings-button") + .should("be.visible") + .click() + + // Then + cy.getByDataHook("ai-promo-modal").should("be.visible") + + // When + cy.getByDataHook("ai-promo-close").should("be.visible").click() + + // Then + cy.getByDataHook("ai-promo-modal").should("not.exist") + + // When + cy.getByDataHook("ai-assistant-settings-button") + .should("be.visible") + .click() + cy.getByDataHook("ai-promo-continue").should("be.visible").click() + + // Then + cy.getByDataHook("ai-settings-modal-step-one").should("be.visible") + }) + + it("should handle invalid api key", () => { + // When + cy.getByDataHook("ai-assistant-settings-button") + .should("be.visible") + .click() + cy.getByDataHook("ai-promo-continue").should("be.visible").click() + + // Then + cy.getByDataHook("ai-settings-modal-step-one").should("be.visible") + // API key input is hidden until a provider is selected + cy.getByDataHook("ai-settings-api-key").should("not.exist") + + // When - select Anthropic + cy.getByDataHook("ai-settings-provider-anthropic").click() + + // Then - API key input appears + cy.getByDataHook("ai-settings-api-key") + .should("be.visible") + .should("have.attr", "placeholder", "Enter Anthropic API key") + + // When - switch to OpenAI + cy.getByDataHook("ai-settings-provider-openai").click() + + // Then + cy.getByDataHook("ai-settings-api-key") + .should("be.visible") + .should("have.attr", "placeholder", "Enter OpenAI API key") + ;["anthropic", "openai"].forEach((provider) => { + // Given + interceptTokenValidation(provider, false) + + // When + cy.getByDataHook(`ai-settings-provider-${provider}`).click() + + // Then + cy.getByDataHook("ai-settings-api-key") + .should("be.visible") + .should( + "have.attr", + "placeholder", + `Enter ${provider === "anthropic" ? "Anthropic" : "OpenAI"} API key`, + ) + .should("be.empty") + + // When + cy.getByDataHook("ai-settings-api-key").type("invalid-api-key") + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then + cy.getByDataHook("multi-step-modal-next-button") + .should("be.disabled") + .should("contain", "Validating...") + + // When + cy.wait(`@${provider}Validation`) + + // Then + cy.getByDataHook("ai-settings-api-key-error").should("be.visible") + }) + }) + + it("should show ai buttons after setup is completed", () => { + // Given + interceptTokenValidation("openai", true) + + // When + cy.getByDataHook("ai-assistant-settings-button") + .should("be.visible") + .click() + cy.getByDataHook("ai-promo-continue").should("be.visible").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("valid-api-key") + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then - step two shows the filtered listing, nothing preselected + cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") + cy.getByDataHook("configure-models-model-row").should("have.length", 4) + + // When - enable two models and activate + cy.getByDataHook("configure-models-model-row").contains("gpt-5.4").click() + cy.getByDataHook("configure-models-model-row") + .contains("gpt-5-mini") + .click() + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then + cy.getByDataHook("ai-assistant-settings-button").should( + "contain", + "AI Settings", + ) + cy.getByDataHook("ai-chat-button").should("be.visible") + cy.getByDataHook("ai-settings-model-dropdown").should("be.visible") + + // When / Then — selecting a model closes the dropdown (handleModelSelect + // calls setDropdownActive(false)), so re-open it for each model and + // re-query the item just before clicking; otherwise the list detaches the + // node as it settles/closes and cy.click() hits a stale element. + ;[0, 1].forEach((index) => { + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.getByDataHook("ai-settings-model-item").should("be.visible") + + cy.getByDataHook("ai-settings-model-item") + .eq(index) + .find("[data-hook='ai-settings-model-item-label']") + .invoke("text") + .then((text) => { + const label = text.trim() + cy.getByDataHook("ai-settings-model-item").eq(index).click() + cy.getByDataHook("ai-settings-model-dropdown").should( + "contain", + label, + ) + }) + }) + + // When + cy.typeQuery("SELECT 1;") + + // Then + cy.getAIIconInLine(1).should("be.visible") + + // When + cy.getByDataHook("ai-assistant-settings-button").click() + + // Then + cy.getByDataHook("ai-settings-validated-badge") + .should("be.visible") + .should("contain", "Validated") + cy.getByDataHook("ai-settings-provider-openai") + .getByDataHook("ai-settings-provider-status") + .should("be.visible") + .should("contain", "Enabled") + + cy.getByDataHook("ai-settings-provider-anthropic") + .getByDataHook("ai-settings-provider-status") + .should("be.visible") + .should("contain", "Inactive") + + // When + cy.getByDataHook("ai-settings-remove-provider").scrollIntoView() + cy.getByDataHook("ai-settings-remove-provider") + .should("be.visible") + .click() + + // Then + cy.getByDataHook("ai-settings-validated-badge").should("not.exist") + cy.getByDataHook("ai-settings-provider-openai") + .getByDataHook("ai-settings-provider-status") + .should("be.visible") + .should("contain", "Inactive") + + // When + cy.getByDataHook("ai-settings-save").click() + + // Then + cy.getByDataHook("ai-settings-model-dropdown").should("not.exist") + cy.getByDataHook("ai-chat-button").should("not.exist") + cy.getByDataHook("ai-assistant-settings-button").should( + "contain", + "Configure", + ) + }) + + it("should not provide schema tools when schema access is disabled", () => { + const schemaTools = ["get_tables", "get_table_schema"] + + // Given + interceptTokenValidation("openai", true) + + // When + cy.getByDataHook("ai-assistant-settings-button") + .should("be.visible") + .click() + cy.getByDataHook("ai-promo-continue").should("be.visible").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("valid-api-key") + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then + cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") + + // When - enable a model, drop permissions to None so schema tools are excluded. + cy.getByDataHook("configure-models-model-row") + .contains("gpt-5-mini") + .click() + cy.getByDataHook("permissions-trigger").click() + cy.getByDataHook("permission-level-none").click() + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then - AI chat should be available + cy.get(".toast-success-container").should("be.visible").click() + cy.getByDataHook("ai-chat-button").should("be.visible") + + // When - Open chat and send a message + interceptAIChatRequest("openai", "chatWithoutSchema") + cy.getByDataHook("ai-chat-button").click() + cy.getByDataHook("ai-chat-window").should("be.visible") + cy.getByDataHook("chat-input-textarea").type("Hello, test message") + cy.getByDataHook("chat-send-button").click() + + // Then - Verify request does NOT contain schema tools + cy.wait("@chatWithoutSchema").then((interception) => { + const tools = interception.request.body.tools || [] + const toolNames = tools.map((t) => t.name || t.function?.name) + schemaTools.forEach((schemaTool) => { + expect(toolNames).to.not.include(schemaTool) + }) + }) + + // When - Open settings modal and re-enable schema access + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("permissions-trigger").click() + cy.getByDataHook("permission-level-schema").click() + cy.getByDataHook("ai-settings-save").click() + cy.get(".toast-success-container").should("be.visible").click() + + // When - Send another message + interceptAIChatRequest("openai", "chatWithSchema") + cy.getByDataHook("chat-input-textarea").type("Another test message") + cy.getByDataHook("chat-send-button").click() + + // Then - Verify request DOES contain schema tools + cy.wait("@chatWithSchema").then((interception) => { + const tools = interception.request.body.tools || [] + const toolNames = tools.map((t) => t.name || t.function?.name) + schemaTools.forEach((schemaTool) => { + expect(toolNames).to.include(schemaTool) + }) + }) + }) + + it("should work with multiple providers", () => { + const openaiEnabledModels = ["GPT 5.4", "GPT 5 Mini"] + const anthropicEnabledModels = ["Claude Opus 4.5", "Claude Sonnet 4.5"] + + // Given - Set up OpenAI provider first + interceptTokenValidation("openai", true) + + // When - Complete setup with OpenAI + cy.getByDataHook("ai-assistant-settings-button") + .should("be.visible") + .click() + cy.getByDataHook("ai-promo-continue").should("be.visible").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("valid-openai-key") + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then - Should be on step two + cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") + + // When - Enable two OpenAI models + cy.getByDataHook("configure-models-model-row").contains("gpt-5.4").click() + cy.getByDataHook("configure-models-model-row") + .contains("gpt-5-mini") + .click() + + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then - Verify model dropdown shows exactly the enabled OpenAI models + cy.get(".toast-success-container").should("be.visible").click() + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.then(() => { + cy.getByDataHook("ai-settings-model-item").should( + "have.length", + openaiEnabledModels.length, + ) + openaiEnabledModels.forEach((modelLabel) => { + cy.getByDataHook("ai-settings-model-item").contains(modelLabel) + }) + }) + cy.get("body").type("{esc}") // close dropdown + + // When - Open settings and configure Anthropic provider + interceptTokenValidation("anthropic", true) + cy.getByDataHook("ai-assistant-settings-button").click() + + // Then - OpenAI should show Enabled, Anthropic should show Inactive + cy.getByDataHook("ai-settings-provider-openai") + .getByDataHook("ai-settings-provider-status") + .should("contain", "Enabled") + cy.getByDataHook("ai-settings-provider-anthropic") + .getByDataHook("ai-settings-provider-status") + .should("contain", "Inactive") + + // When - Configure Anthropic + cy.getByDataHook("ai-settings-provider-anthropic").click() + cy.getByDataHook("ai-settings-api-key").type("valid-anthropic-key") + cy.getByDataHook("ai-settings-test-api").click() + + // Then - Validation opens Manage Models with the fetched listing + cy.wait("@anthropicValidation") + cy.getByDataHook("manage-models-model-row").should("have.length", 3) + + // When - Enable two Anthropic models and save the picker + cy.getByDataHook("manage-models-model-row") + .contains("Claude Opus 4.5") + .click() + cy.getByDataHook("manage-models-model-row") + .contains("Claude Sonnet 4.5") + .click() + cy.getByDataHook("manage-models-save").click() + + // Then - Anthropic should no longer show Inactive + cy.getByDataHook("ai-settings-provider-anthropic") + .getByDataHook("ai-settings-provider-status") + .should("not.contain", "Inactive") + + // When - Save settings (the picker's own save may still show its toast) + cy.getByDataHook("ai-settings-save").click() + cy.get(".toast-success-container") + .should("be.visible") + .click({ multiple: true }) + + // Then - Model dropdown should contain models from both providers + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.then(() => { + const allEnabledModels = [ + ...openaiEnabledModels, + ...anthropicEnabledModels, + ] + cy.getByDataHook("ai-settings-model-item").should( + "have.length", + allEnabledModels.length, + ) + allEnabledModels.forEach((modelLabel) => { + cy.getByDataHook("ai-settings-model-item").contains(modelLabel) + }) + }) + + // When - Select first OpenAI model and open chat + cy.then(() => { + cy.getByDataHook("ai-settings-model-item") + .contains(openaiEnabledModels[0]) + .click() + }) + interceptAIChatRequest("openai", "openaiChat") + cy.getByDataHook("ai-chat-button").click() + cy.getByDataHook("ai-chat-window").should("be.visible") + cy.getByDataHook("chat-input-textarea").type("Test message for OpenAI") + cy.getByDataHook("chat-send-button").click() + + // Then - Should intercept OpenAI request + cy.wait("@openaiChat") + + // When - Select first Anthropic model from dropdown + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.then(() => { + cy.getByDataHook("ai-settings-model-item") + .contains(anthropicEnabledModels[0]) + .click() + }) + + // When - Send another message + interceptAIChatRequest("anthropic", "anthropicChat") + cy.getByDataHook("chat-input-textarea").type("Test message for Anthropic") + cy.getByDataHook("chat-send-button").click() + + // Then - Should intercept Anthropic request + cy.wait("@anthropicChat") + }) + }) +}) diff --git a/e2e/utils/aiAssistant.js b/e2e/utils/aiAssistant.js index c3bbecbe1..d9c89be96 100644 --- a/e2e/utils/aiAssistant.js +++ b/e2e/utils/aiAssistant.js @@ -22,11 +22,12 @@ const CUSTOM_PROVIDER_DEFAULTS = { function getOpenAIConfiguredSettings(schemaAccess = true) { return { "ai.assistant.settings": JSON.stringify({ - selectedModel: "gpt-5-mini", + modelValueFormat: 2, + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "test-openai-key", - enabledModels: ["gpt-5-mini", "gpt-5"], + enabledModels: ["openai:gpt-5-mini", "openai:gpt-5"], grantSchemaAccess: schemaAccess, }, }, @@ -48,11 +49,12 @@ function getOpenAIPermissionedSettings({ }) { return { "ai.assistant.settings": JSON.stringify({ - selectedModel: "gpt-5-mini", + modelValueFormat: 2, + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "test-openai-key", - enabledModels: ["gpt-5-mini", "gpt-5"], + enabledModels: ["openai:gpt-5-mini", "openai:gpt-5"], grantSchemaAccess, read, write, @@ -65,11 +67,15 @@ function getOpenAIPermissionedSettings({ function getAnthropicConfiguredSettings(schemaAccess = true) { return { "ai.assistant.settings": JSON.stringify({ - selectedModel: "claude-sonnet-4-5", + modelValueFormat: 2, + selectedModel: "anthropic:claude-sonnet-4-5", providers: { anthropic: { apiKey: "test-anthropic-key", - enabledModels: ["claude-sonnet-4-5", "claude-opus-4-5"], + enabledModels: [ + "anthropic:claude-sonnet-4-5", + "anthropic:claude-opus-4-5", + ], grantSchemaAccess: schemaAccess, }, }, @@ -101,6 +107,7 @@ function getCustomProviderConfiguredSettings(config = {}, mergeWith = null) { const settings = { ...baseSettings, + modelValueFormat: 2, selectedModel: enabledModels[0], customProviders: { ...(baseSettings.customProviders || {}), @@ -1163,7 +1170,42 @@ function createMultiTurnFlow(config) { } } +/** + * Intercepts AI chat requests with a default test response. + * + * @param {"anthropic" | "openai"} provider - The AI provider to intercept + * @param {string} [alias] - Optional custom alias for the intercept + * @param {number} [delay=200] - Delay in milliseconds + * @param {Object} [options] - Options + * @param {boolean} [options.streaming=true] - Whether to use streaming response + */ +function interceptAIChatRequest( + provider, + alias, + delay = 200, + options = { streaming: true }, +) { + const aliasName = alias || `${provider}ChatRequest` + const endpoint = PROVIDERS[provider].endpoint + const { streaming = true } = options + + const responseData = createFinalResponseData( + provider, + "Test response explanation", + ) + + cy.intercept("POST", endpoint, (req) => { + if (isTitleRequest(provider, req.body)) { + req.reply(createChatTitleResponse(provider, "Test Chat")) + return + } + req.alias = aliasName + req.reply(createResponse(provider, responseData, { streaming, delay })) + }) +} + module.exports = { + interceptAIChatRequest, PROVIDERS, CUSTOM_PROVIDER_DEFAULTS, getOpenAIConfiguredSettings, diff --git a/src/components/AIStatusIndicator/index.tsx b/src/components/AIStatusIndicator/index.tsx index ec0cdf68c..66a2fa4d7 100644 --- a/src/components/AIStatusIndicator/index.tsx +++ b/src/components/AIStatusIndicator/index.tsx @@ -11,11 +11,9 @@ import { color } from "../../utils" import { slideAnimation } from "../Animation" import { AISparkle } from "../AISparkle" import { brandLinearGradientHorizontal } from "../../theme" -import { getAllModelOptions } from "../../utils/ai" import { useAIConversation } from "../../providers/AIConversationProvider" import { Button } from "../../components/Button" import { AIStopButton } from "../AIStopButton" -import { BrainIcon } from "../SetupAIAssistant/BrainIcon" import { AssistantModes, buildOperationSections } from "./AssistantModes" import { CircleNotchSpinner } from "../../scenes/Editor/Monaco/icons" import { useSelector } from "react-redux" @@ -220,34 +218,6 @@ const ChevronButton = styled(Button).attrs({ variant: "ghost" })` margin-right: 1rem; ` -const ExtendedThinkingLabel = styled.div` - display: flex; - gap: 0.8rem; - align-items: center; - justify-content: center; - width: 100%; - flex-shrink: 0; -` - -const BrainIconWrapper = styled.div` - width: 1.6rem; - height: 1.6rem; - flex-shrink: 0; - display: flex; - align-items: center; - justify-content: center; -` - -const ExtendedThinkingText = styled.p` - flex: 1 0 0; - font-weight: 400; - font-size: 1.1rem; - color: ${color("contentSecondary")}; - min-height: 0; - min-width: 0; - margin: 0; -` - const AssistantModesContainer = styled.div` display: flex; flex-direction: column; @@ -264,14 +234,8 @@ const AssistantModesContainer = styled.div` ` export const AIStatusIndicator: React.FC = () => { - const { - status, - currentOperation, - currentModel, - abortOperation, - clearOperation, - aiAssistantSettings, - } = useAIStatus() + const { status, currentOperation, abortOperation, clearOperation } = + useAIStatus() const { chatWindowState, openChatWindow } = useAIConversation() const [expanded, setExpanded] = useState(false) const [isClosed, setIsClosed] = useState(false) @@ -280,11 +244,6 @@ export const AIStatusIndicator: React.FC = () => { const assistantModesRef = useRef(null) const activeSidebar = useSelector(selectors.console.getActiveSidebar) const statusRef = useRef(null) - const hasExtendedThinking = useMemo(() => { - return getAllModelOptions(aiAssistantSettings).find( - (model) => model.value === currentModel, - )?.isSlow - }, [currentModel, aiAssistantSettings]) const operationSections = useMemo( () => buildOperationSections(currentOperation, status, true), @@ -416,17 +375,6 @@ export const AIStatusIndicator: React.FC = () => { )} - {hasExtendedThinking && ( - - - - - - Extended thinking model enabled. Responses may be slow. - - - )} - {expanded && ( theme.color.statusDanger}; - font-size: 1.3rem; - text-align: right; - width: 100%; -` - const CancelButton = styled(Button)` flex: 1; padding: 1.1rem 1.2rem; @@ -186,7 +186,6 @@ type MultiStepModalProps = { canProceed?: (stepIndex: number) => boolean | Promise completeButtonText?: string onStepChange?: (stepIndex: number, direction: "next" | "previous") => void - showValidationError?: boolean } export const MultiStepModal = ({ @@ -199,18 +198,20 @@ export const MultiStepModal = ({ canProceed, completeButtonText = "Complete", onStepChange, - showValidationError = true, }: MultiStepModalProps) => { const [currentStep, setCurrentStep] = useState(0) const [validationError, setValidationError] = useState(null) const [isValidating, setIsValidating] = useState(false) + const sessionRef = useRef(0) + const handleOpenChange = (isOpen: boolean) => { if (!isOpen && onCancel) { onCancel() } onOpenChange?.(isOpen) if (!isOpen) { + sessionRef.current += 1 setCurrentStep(0) setValidationError(null) setIsValidating(false) @@ -225,19 +226,21 @@ export const MultiStepModal = ({ } if (currentStepData?.validate) { + const session = sessionRef.current setValidationError(null) setIsValidating(true) try { const validationResult = await currentStepData.validate() + if (session !== sessionRef.current) return if (typeof validationResult === "string") { setValidationError(validationResult) return } else if (validationResult === false) { - setValidationError("Validation failed") return } } catch (error) { + if (session !== sessionRef.current) return const errorMessage = error instanceof Error ? error.message : "Validation failed" setValidationError(errorMessage) @@ -323,10 +326,12 @@ export const MultiStepModal = ({ ? steps[currentStep]?.content() : steps[currentStep]?.content} + {validationError && ( + + {validationError} + + )} - {showValidationError && validationError && ( - {validationError} - )} theme.color.contentPrimary}; ` -const ModelToggleRow = styled(Box).attrs({ - justifyContent: "space-between", - align: "center", - gap: "2.4rem", -})` - width: 100%; -` - -const ModelInfoColumn = styled(Box).attrs({ - flexDirection: "column", - gap: "0.8rem", -})` - flex: 1; - align-items: flex-start; -` - -const ModelInfoRow = styled(Box).attrs({ - gap: "0.8rem", - align: "center", -})` - width: 100%; -` - -const ModelDescriptionText = styled(Text)` - font-size: 1.1rem; - color: ${({ theme }) => theme.color.contentSecondary}; - flex: 1; -` - -const ModelNameText = styled(Text)` - font-size: 1.4rem; - font-weight: 400; - color: ${({ theme }) => theme.color.contentPrimary}; -` - const WarningText = styled(Text)` font-size: 1.3rem; font-weight: 400; @@ -280,10 +246,14 @@ type StepOneContentProps = { type StepTwoContentProps = { selectedProvider: ProviderId | null + listing: ProviderModel[] | null enabledModels: string[] + manualInput: string + reasoningEffortLevel: ReasoningEffortLevel permissions: Permissions - modelsByProvider: Record - onModelToggle: (modelValue: string) => void + onSelectionChange: (models: string[]) => void + onManualInputChange: (value: string) => void + onReasoningEffortChange: (next: ReasoningEffortLevel) => void onPermissionsChange: (next: Permissions) => void } @@ -398,9 +368,11 @@ const StepOneContent = ({ data-hook="ai-settings-api-key" /> {error && ( - - {error} - + + + {error} + + )} Stored locally in your browser and never sent to QuestDB @@ -417,10 +389,14 @@ const StepOneContent = ({ const StepTwoContent = ({ selectedProvider, + listing, enabledModels, + manualInput, + reasoningEffortLevel, permissions, - modelsByProvider, - onModelToggle, + onSelectionChange, + onManualInputChange, + onReasoningEffortChange, onPermissionsChange, }: StepTwoContentProps) => { const theme = useTheme() @@ -428,9 +404,12 @@ const StepTwoContent = ({ const handleClose: () => void = navigation.handleClose const currentProvider = selectedProvider - const getModelsForProvider = (provider: ProviderId) => { - return modelsByProvider[provider] || [] - } + const isOpenAi = currentProvider === "openai" + const pickerModels = listing + ? isOpenAi + ? filterOpenAiChatModels(listing) + : sortModelsNewestFirst(listing) + : [] return ( @@ -439,9 +418,9 @@ const StepTwoContent = ({ Setup your model preferences - Enable and disable each of the models QuestDB currently supports - from this provider, and a level of data access. You'll be - able to update these settings any time. + Enable the models you want to use from this provider, and a level + of data access. You'll be able to update these settings any + time. @@ -449,11 +428,11 @@ const StepTwoContent = ({ - {currentProvider ? ( + {currentProvider && listing ? ( - Enable Models + Models {currentProvider === "anthropic" ? ( - - {getModelsForProvider(currentProvider).map((model) => { - const isEnabled = enabledModels.includes(model.value) - return ( - - - {model.label} - {model.isSlow && ( - - - - Due to advanced reasoning & thinking - capabilities, responses using this model can be - slow. - - - )} - - onModelToggle(model.value)} - data-checked={isEnabled} - /> - - ) - })} - + model.label ?? formatModelLabel(model.id)} + onSelectionChange={onSelectionChange} + onManualInputChange={onManualInputChange} + /> ) : ( @@ -515,6 +472,17 @@ const StepTwoContent = ({ )} + {isOpenAi && ( + <> + + + + + + )} {currentProvider && ( @@ -539,6 +507,7 @@ export const ConfigurationModal = ({ onOpenChange, }: ConfigurationModalProps) => { const { aiAssistantSettings, updateSettings } = useLocalStorage() + const closeCountRef = useRef(0) const [selectedProvider, setSelectedProvider] = useState( null, ) @@ -557,20 +526,15 @@ export const ConfigurationModal = ({ }, [open]) const [enabledModels, setEnabledModels] = useState([]) + const [manualInput, setManualInput] = useState("") + const [providerListing, setProviderListing] = useState< + ProviderModel[] | null + >(null) + const [reasoningEffortLevel, setReasoningEffortLevel] = + useState("default") const [permissions, setPermissions] = useState(DEFAULT_PERMISSIONS) - const modelsByProvider = useMemo(() => { - const result: Record = {} - getAllModelOptions(aiAssistantSettings).forEach((model) => { - if (!result[model.provider]) { - result[model.provider] = [] - } - result[model.provider].push(model) - }) - return result - }, [aiAssistantSettings]) - const handleProviderSelect = useCallback((provider: ProviderId) => { setSelectedProvider(provider) setError(null) @@ -582,21 +546,23 @@ export const ConfigurationModal = ({ setError(null) }, []) - const handleModelToggle = useCallback((modelValue: string) => { - setEnabledModels((prev) => { - const isEnabled = prev.includes(modelValue) - return isEnabled - ? prev.filter((m) => m !== modelValue) - : [...prev, modelValue] - }) - }, []) - const handlePermissionsChange = useCallback((next: Permissions) => { setPermissions(next) }, []) + const effectiveEnabledModels = useCallback(() => { + const pending = manualInput.trim() + return pending && !enabledModels.includes(pending) + ? [...enabledModels, pending] + : enabledModels + }, [enabledModels, manualInput]) + const handleComplete = () => { - if (!selectedProvider || enabledModels.length === 0) return + const models = effectiveEnabledModels() + if (!selectedProvider || models.length === 0) return + const modelValues = models.map((model) => + makeModelValue(selectedProvider, model), + ) void trackEvent(ConsoleEvent.AI_PROVIDER_CONFIGURE, { name: selectedProvider, @@ -605,25 +571,23 @@ export const ConfigurationModal = ({ write: permissions.write, }) - const selectedModel = - enabledModels.find( - (m) => - getAllModelOptions(aiAssistantSettings).find((mo) => mo.value === m) - ?.default, - ) ?? enabledModels[0] + const metadata = providerListing + ? buildListingMetadata(selectedProvider, providerListing, models) + : null const newSettings = { ...aiAssistantSettings, - selectedModel, + selectedModel: modelValues[0], providers: { ...aiAssistantSettings.providers, - [selectedProvider]: { + [selectedProvider]: buildProviderSettings({ apiKey, - enabledModels, - grantSchemaAccess: permissions.grantSchemaAccess, - read: permissions.read, - write: permissions.write, - }, + enabledModels: modelValues, + permissions, + modelLabels: metadata?.modelLabels, + utilityModel: metadata?.utilityModel, + reasoningEffort: reasoningEffortLevel, + }), }, } @@ -649,53 +613,46 @@ export const ConfigurationModal = ({ return "Please enter an API key" } - const testModel = - getAllModelOptions(aiAssistantSettings).find( - (m) => m.isTestModel && m.provider === selectedProvider, - )?.value ?? modelsByProvider[selectedProvider][0].value - + const provider = createProvider( + selectedProvider, + apiKey, + aiAssistantSettings, + ) + const session = closeCountRef.current try { - const result = await testApiKey( - apiKey, - testModel, - selectedProvider, - aiAssistantSettings, - ) - if (!result.valid) { - const errorMsg = result.error || "Invalid API key" - setError(errorMsg) - return errorMsg - } - const defaultModels = getAllModelOptions(aiAssistantSettings) - .filter((m) => m.defaultEnabled && m.provider === selectedProvider) - .map((m) => m.value) - if (defaultModels.length > 0) { - setEnabledModels(defaultModels) - } + const listing = await provider.listModels() + if (session !== closeCountRef.current) return false + setProviderListing(listing) setError(null) void trackEvent(ConsoleEvent.AI_CONFIGURATION_VALIDATE) return true } catch (err) { + const classified = provider.classifyError(err, () => {}) const errorMessage = - err instanceof Error ? err.message : "Failed to validate API key" + classified.type === "invalid_key" + ? "Invalid API key" + : classified.message setError(errorMessage) - return errorMessage + return false } - }, [selectedProvider, apiKey, modelsByProvider]) + }, [selectedProvider, apiKey, aiAssistantSettings]) const validateStepTwo = useCallback((): string | boolean => { if (!selectedProvider) return "Please select a provider" - if (enabledModels.length === 0) { + if (effectiveEnabledModels().length === 0) { return "Please enable at least one model" } return true - }, [enabledModels, selectedProvider]) + }, [effectiveEnabledModels, selectedProvider]) const handleStepChange = useCallback( (newStepIndex: number, direction: "next" | "previous") => { // When going back from step 2 to step 1, reset step 2 state but keep API key if (newStepIndex === 0 && direction === "previous") { setEnabledModels([]) + setManualInput("") + setProviderListing(null) + setReasoningEffortLevel("default") setPermissions(DEFAULT_PERMISSIONS) } }, @@ -703,17 +660,21 @@ export const ConfigurationModal = ({ ) const handleModalClose = useCallback(() => { + closeCountRef.current += 1 setSelectedProvider(null) setApiKey("") setError(null) setEnabledModels([]) + setManualInput("") + setProviderListing(null) + setReasoningEffortLevel("default") setPermissions(DEFAULT_PERMISSIONS) }, []) const handleCustomProviderSave = useCallback( (providerId: string, definition: CustomProviderDefinition) => { const newEnabledModels = definition.models.map((m) => - makeCustomModelValue(providerId, m), + makeModelValue(providerId, m), ) const newSettings = { @@ -778,10 +739,14 @@ export const ConfigurationModal = ({ content: ( ), @@ -795,10 +760,11 @@ export const ConfigurationModal = ({ providerName, handleProviderSelect, handleApiKeyChange, + providerListing, enabledModels, + manualInput, + reasoningEffortLevel, permissions, - modelsByProvider, - handleModelToggle, handlePermissionsChange, validateStepOne, validateStepTwo, @@ -821,7 +787,6 @@ export const ConfigurationModal = ({ onComplete={handleComplete} canProceed={canProceed} completeButtonText="Activate Assistant" - showValidationError={false} /> {customProviderModalOpen && ( theme.color.statusDanger}; ` +const ContentSection = styled(Box).attrs({ + flexDirection: "column", + gap: "2rem", +})` + padding: 2.4rem; + width: 100%; +` + +const LoadingContainer = styled(Box).attrs({ + align: "center", + justifyContent: "center", +})` + width: 100%; + padding: 4rem 0; +` + +export type BuiltinModelsResult = { + enabledModels: string[] + modelLabels: Record + utilityModel?: string +} + +type BuiltinModelsRef = { + getResult: () => BuiltinModelsResult | null + validate: () => string | true +} + +type BuiltinModelsContentProps = { + providerId: string + apiKey: string + enabledModels: string[] + initialListing?: ProviderModel[] + onLoadingChange: (loading: boolean) => void + onFetchFailedChange: (failed: boolean) => void +} + +const BuiltinModelsContent = forwardRef< + BuiltinModelsRef, + BuiltinModelsContentProps +>( + ( + { + providerId, + apiKey, + enabledModels, + initialListing, + onLoadingChange, + onFetchFailedChange, + }, + ref, + ) => { + const [listing, setListing] = useState( + initialListing ?? null, + ) + const [fetchError, setFetchError] = useState(null) + const [selectedModels, setSelectedModels] = useState( + initialListing ? enabledModels : [], + ) + const [manualInput, setManualInput] = useState("") + const [isLoading, setIsLoading] = useState(!initialListing) + + const isOpenAi = BUILTIN_PROVIDERS[providerId]?.type === "openai" + const pickerModels = listing + ? isOpenAi + ? filterOpenAiChatModels(listing) + : sortModelsNewestFirst(listing) + : [] + + const selectionWithPending = () => { + const pending = manualInput.trim() + return pending && !selectedModels.includes(pending) + ? [...selectedModels, pending] + : [...selectedModels] + } + + useImperativeHandle( + ref, + () => ({ + getResult: () => { + if (!listing) return null + const models = selectionWithPending() + return { + enabledModels: models, + ...buildListingMetadata(providerId, listing, models), + } + }, + validate: () => { + if (!listing) return "Could not fetch models from the provider" + if (selectionWithPending().length === 0) + return "Enable at least one model" + return true + }, + }), + [listing, selectedModels, manualInput, providerId], + ) + + useEffect(() => { + let cancelled = false + + const doFetch = async () => { + if (initialListing) { + onFetchFailedChange(false) + onLoadingChange(false) + return + } + onFetchFailedChange(false) + onLoadingChange(true) + const provider = createProviderByType( + BUILTIN_PROVIDERS[providerId].type, + providerId, + apiKey, + ) + try { + const models = await provider.listModels() + if (cancelled) return + setListing(models) + setSelectedModels(enabledModels) + } catch (error) { + if (cancelled) return + const classified = provider.classifyError(error, () => {}) + setFetchError( + classified.type === "rate_limit" + ? classified.message + : "Could not fetch models from the provider. Check your API key and connection, then try again.", + ) + onFetchFailedChange(true) + } finally { + if (!cancelled) { + setIsLoading(false) + onLoadingChange(false) + } + } + } + + void doFetch() + return () => { + cancelled = true + } + }, []) + + if (isLoading) { + return ( + + + + + + ) + } + + if (fetchError) { + return ( + + + {fetchError} + + + ) + } + + return ( + + model.label ?? formatModelLabel(model.id)} + onSelectionChange={setSelectedModels} + onManualInputChange={setManualInput} + /> + + ) + }, +) + +BuiltinModelsContent.displayName = "BuiltinModelsContent" + type ManageModelsModalProps = { open: boolean onOpenChange: (open: boolean) => void providerId: string - definition: CustomProviderDefinition - onSave: (providerId: string, definition: CustomProviderDefinition) => void -} +} & ( + | { + variant: "custom" + definition: CustomProviderDefinition + onSave: (providerId: string, definition: CustomProviderDefinition) => void + } + | { + variant: "builtin" + apiKey: string + enabledModels: string[] + initialListing?: ProviderModel[] + onSave: (providerId: string, result: BuiltinModelsResult) => void + } +) -export const ManageModelsModal = ({ - open, - onOpenChange, - providerId, - definition, - onSave, -}: ManageModelsModalProps) => { +export const ManageModelsModal = (props: ManageModelsModalProps) => { + const { open, onOpenChange, providerId } = props const [error, setError] = useState(null) const [modelsLoading, setModelsLoading] = useState(true) + const [modelsFetchFailed, setModelsFetchFailed] = useState(false) const modelSettingsRef = useRef(null) + const builtinModelsRef = useRef(null) + + const providerName = + props.variant === "custom" + ? props.definition.name + : getProviderName(providerId) const handleSave = useCallback(() => { setError(null) - const result = modelSettingsRef.current?.validate() - if (typeof result === "string") { - setError(result) - return + if (props.variant === "custom") { + const result = modelSettingsRef.current?.validate() + if (typeof result === "string") { + setError(result) + return + } + const values = modelSettingsRef.current?.getValues() + if (!values) return + props.onSave(providerId, { + ...props.definition, + models: values.models, + contextWindow: values.contextWindow, + }) + } else { + const result = builtinModelsRef.current?.validate() + if (typeof result === "string") { + setError(result) + return + } + const values = builtinModelsRef.current?.getResult() + if (!values) return + props.onSave(providerId, values) } - const values = modelSettingsRef.current?.getValues() - if (!values) return - onSave(providerId, { - ...definition, - models: values.models, - contextWindow: values.contextWindow, - }) onOpenChange(false) - }, [definition, providerId, onSave, onOpenChange]) + }, [props, providerId, onOpenChange]) return ( @@ -120,37 +341,54 @@ export const ManageModelsModal = ({ Manage Models - Add or remove models and update the context window for{" "} - {definition.name}. + {props.variant === "custom" + ? `Add or remove models and update the context window for ${providerName}.` + : `Enable the ${providerName} models you want to use.`} - {open && ( + {open && props.variant === "custom" && ( )} + {open && props.variant === "builtin" && ( + + )} - + {error ? ( + + {error} + + ) : ( + + )} - {error && {error}} Save diff --git a/src/components/SetupAIAssistant/ModelDropdown.tsx b/src/components/SetupAIAssistant/ModelDropdown.tsx index bcd2478cf..3412d0854 100644 --- a/src/components/SetupAIAssistant/ModelDropdown.tsx +++ b/src/components/SetupAIAssistant/ModelDropdown.tsx @@ -6,7 +6,6 @@ import { useAIStatus } from "../../providers/AIStatusProvider" import { StoreKey } from "../../utils/localStorage/types" import { OpenAIIcon } from "./OpenAIIcon" import { AnthropicIcon } from "./AnthropicIcon" -import { BrainIcon } from "./BrainIcon" import { PlugsIcon, WarningCircleIcon } from "@phosphor-icons/react" import { SelectMenu } from "../SelectMenu" import { trackEvent } from "../../modules/ConsoleEventTracker" @@ -43,7 +42,6 @@ export const ModelDropdown = () => { return null } - // currentModel is guaranteed to be from MODEL_OPTIONS (set in modals) const displayModel = currentModel ? (enabledModels.find((m) => m.value === currentModel) ?? enabledModels[0]) : (enabledModels[0] ?? null) @@ -106,7 +104,6 @@ export const ModelDropdown = () => { > {model.label} - {model.isSlow && } ))} diff --git a/src/components/SetupAIAssistant/ModelPicker.tsx b/src/components/SetupAIAssistant/ModelPicker.tsx new file mode 100644 index 000000000..28d02d129 --- /dev/null +++ b/src/components/SetupAIAssistant/ModelPicker.tsx @@ -0,0 +1,270 @@ +import React from "react" +import styled from "styled-components" +import { XIcon } from "@phosphor-icons/react" +import { Box } from "../Box" +import { Button } from "../Button" +import { Checkbox } from "../Checkbox" +import { IconButton } from "../IconButton" +import { Input } from "../Input" +import { Text } from "../Text" +import { TextButton } from "../TextButton" +import type { ProviderModel } from "../../utils/ai" + +const PickerSection = styled(Box).attrs({ + flexDirection: "column", + gap: "1.2rem", +})` + width: 100%; +` + +const HeaderRow = styled(Box).attrs({ + flexDirection: "row", + gap: "1.2rem", + align: "center", +})` + width: 100%; +` + +const HeaderLabel = styled(Text)` + font-size: 1.6rem; + font-weight: 600; + color: ${({ theme }) => theme.color.contentSecondary}; +` + +const SelectAllRow = styled(Box).attrs({ + gap: "2rem", + align: "center", +})` + display: inline-flex; + margin-left: auto; +` + +const SelectAllLink = styled(TextButton)` + font-size: 1.4rem; +` + +const ModelListContainer = styled.div` + max-height: 30rem; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 0.25rem; + border: 0.1rem solid ${({ theme }) => theme.color.borderStrong}; + border-radius: 0.4rem; + width: 100%; +` + +const ModelRow = styled.label` + display: flex; + align-items: center; + gap: 0.8rem; + padding: 0.6rem 0.8rem; + cursor: pointer; + font-size: 1.4rem; + color: ${({ theme }) => theme.color.contentPrimary}; + + &:hover { + background: ${({ theme }) => theme.color.interactionNeutral}; + } +` + +const ModelIdText = styled(Text)` + font-size: 1.2rem; + color: ${({ theme }) => theme.color.contentSecondary}; +` + +const HelperText = styled(Text)` + font-size: 1.3rem; + font-weight: 400; + color: ${({ theme }) => theme.color.contentSecondary}; +` + +const AddModelRow = styled(Box).attrs({ + gap: "0.8rem", + align: "center", +})` + width: 100%; +` + +const AddModelInput = styled(Input)` + width: 100%; +` + +const AddModelButton = styled(Button).attrs({ variant: "secondary" })` + height: 3rem; + padding: 0 1.2rem; + font-size: 1.4rem; + white-space: nowrap; +` + +const ModelChipsContainer = styled.div` + display: flex; + flex-wrap: wrap; + gap: 0.6rem; +` + +const ModelChip = styled.div` + display: inline-flex; + align-items: center; + gap: 0.5rem; + background: ${({ theme }) => theme.color.interactionNeutral}; + border-radius: 0.4rem; + padding: 0.4rem 0.8rem; + font-size: 1.3rem; + color: ${({ theme }) => theme.color.contentPrimary}; +` + +const ChipRemoveButton = styled(IconButton)` + padding: 0; + width: 2rem; + min-width: 2rem; + height: 2rem; +` + +export type ModelPickerProps = { + listedModels: ProviderModel[] + selectedModels: string[] + manualInput: string + dataHookPrefix: string + labelFor?: (model: ProviderModel) => string + onSelectionChange: (models: string[]) => void + onManualInputChange: (value: string) => void +} + +export const ModelPicker = ({ + listedModels, + selectedModels, + manualInput, + dataHookPrefix, + labelFor, + onSelectionChange, + onManualInputChange, +}: ModelPickerProps) => { + const isRowChecked = (rowId: string) => selectedModels.includes(rowId) + const manualModels = selectedModels.filter( + (selected) => !listedModels.some((m) => m.id === selected), + ) + + const handleToggleRow = (rowId: string) => { + if (isRowChecked(rowId)) { + onSelectionChange(selectedModels.filter((s) => s !== rowId)) + } else { + onSelectionChange([...selectedModels, rowId]) + } + } + + const handleSelectAll = () => { + const unchecked = listedModels + .filter((m) => !isRowChecked(m.id)) + .map((m) => m.id) + onSelectionChange([...selectedModels, ...unchecked]) + } + + const handleDeselectAll = () => { + onSelectionChange( + selectedModels.filter((s) => !listedModels.some((m) => m.id === s)), + ) + } + + const handleAddManualModel = () => { + const trimmed = manualInput.trim() + if (!trimmed) return + if (!selectedModels.includes(trimmed)) { + onSelectionChange([...selectedModels, trimmed]) + } + onManualInputChange("") + } + + const handleRemoveManualModel = (model: string) => { + onSelectionChange(selectedModels.filter((m) => m !== model)) + } + + return ( + <> + + + Select Models + + + Select All + + + Deselect All + + + + + {listedModels.map((model) => { + const label = labelFor ? labelFor(model) : model.id + return ( + + handleToggleRow(model.id)} + /> + {label} + {label !== model.id && {model.id}} + + ) + })} + + + + Don't see your model? Add it manually: + + onManualInputChange(e.target.value)} + placeholder="e.g., llama3, gpt-4o, claude-sonnet-4-20250514" + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault() + handleAddManualModel() + } + }} + /> + + Add + + + {manualModels.length > 0 && ( + + {manualModels.map((model) => ( + + {model} + handleRemoveManualModel(model)} + > + + + + ))} + + )} + + + ) +} diff --git a/src/components/SetupAIAssistant/ModelSettings.tsx b/src/components/SetupAIAssistant/ModelSettings.tsx index 233ac017e..24be0a8c1 100644 --- a/src/components/SetupAIAssistant/ModelSettings.tsx +++ b/src/components/SetupAIAssistant/ModelSettings.tsx @@ -9,15 +9,15 @@ import React, { import styled, { useTheme } from "styled-components" import { Box } from "../Box" import { Input } from "../Input" -import { Checkbox } from "../Checkbox" import { Text } from "../Text" import { LoadingSpinner } from "../LoadingSpinner" import { Button } from "../Button" import { IconButton } from "../IconButton" -import { TextButton } from "../TextButton" import { WarningIcon, XIcon } from "@phosphor-icons/react" import { createProviderByType } from "../../utils/ai/registry" import type { ProviderType } from "../../utils/ai/settings" +import type { ProviderModel } from "../../utils/ai" +import { ModelPicker } from "./ModelPicker" import { PermissionsSection } from "../../scenes/Footer/MCPBridgeStatus/PermissionsSection" import type { Permissions } from "../../utils/tools/permissions" @@ -61,31 +61,6 @@ const WarningText = styled(Text)` color: ${({ theme }) => theme.color.statusWarning}; ` -const ModelListContainer = styled.div` - max-height: 30rem; - overflow-y: auto; - display: flex; - flex-direction: column; - gap: 0.25rem; - border: 0.1rem solid ${({ theme }) => theme.color.borderStrong}; - border-radius: 0.4rem; - width: 100%; -` - -const ModelRow = styled.label` - display: flex; - align-items: center; - gap: 0.8rem; - padding: 0.6rem 0.8rem; - cursor: pointer; - font-size: 1.4rem; - color: ${({ theme }) => theme.color.contentPrimary}; - - &:hover { - background: ${({ theme }) => theme.color.interactionNeutral}; - } -` - const ModelChipsContainer = styled.div` display: flex; flex-wrap: wrap; @@ -124,18 +99,6 @@ const AddModelButton = styled(Button).attrs({ variant: "secondary" })` white-space: nowrap; ` -const SelectAllRow = styled(Box).attrs({ - gap: "2rem", - align: "center", -})` - display: inline-flex; - margin-left: auto; -` - -const SelectAllLink = styled(TextButton)` - font-size: 1.4rem; -` - const ContentSection = styled(Box).attrs({ flexDirection: "column", gap: "2rem", @@ -193,7 +156,7 @@ export type ModelSettingsProps = { async function fetchProviderModels( config: FetchConfig, contextWindow: number, -): Promise { +): Promise { try { const provider = createProviderByType( config.providerType, @@ -215,7 +178,9 @@ export const ModelSettings = forwardRef( ) => { const theme = useTheme() - const [fetchedModels, setFetchedModels] = useState(null) + const [fetchedModels, setFetchedModels] = useState( + null, + ) const [selectedModels, setSelectedModels] = useState([]) const [manualModels, setManualModels] = useState( () => initialValues?.models ?? [], @@ -252,13 +217,8 @@ export const ModelSettings = forwardRef( if (cancelled) return if (models) { - // Auto mode: reconcile initialValues.models against fetched list setFetchedModels(models) - const selected = [ - ...initModels.filter((m) => models.includes(m)), - ...initModels.filter((m) => !models.includes(m)), - ] - setSelectedModels(selected.length > 0 ? selected : []) + setSelectedModels([...initModels]) setManualModels([]) } else { // Manual mode @@ -281,43 +241,14 @@ export const ModelSettings = forwardRef( const isAutoMode = fetchedModels !== null - const handleToggleModel = useCallback((model: string) => { - setSelectedModels((prev) => - prev.includes(model) - ? prev.filter((m) => m !== model) - : [...prev, model], - ) - }, []) - - const handleSelectAll = useCallback(() => { - setSelectedModels((prev) => { - if (!fetchedModels) return prev - const manual = prev.filter((m) => !fetchedModels.includes(m)) - return [...fetchedModels, ...manual] - }) - }, [fetchedModels]) - - const handleDeselectAll = useCallback(() => { - setSelectedModels((prev) => - fetchedModels ? prev.filter((m) => !fetchedModels.includes(m)) : [], - ) - }, [fetchedModels]) - const handleAddManualModel = useCallback(() => { const trimmed = manualModelInput.trim() if (!trimmed) return - - if (isAutoMode) { - setSelectedModels((prev) => - prev.includes(trimmed) ? prev : [...prev, trimmed], - ) - } else { - setManualModels((prev) => - prev.includes(trimmed) ? prev : [...prev, trimmed], - ) - } + setManualModels((prev) => + prev.includes(trimmed) ? prev : [...prev, trimmed], + ) setManualModelInput("") - }, [manualModelInput, isAutoMode]) + }, [manualModelInput]) const handleRemoveManualModel = useCallback((model: string) => { setManualModels((prev) => prev.filter((m) => m !== model)) @@ -397,118 +328,64 @@ export const ModelSettings = forwardRef( )} {isAutoMode && ( - - - Select Models - - - Select All - - - Deselect All - - - - - {fetchedModels.map((model) => ( - - handleToggleModel(model)} - /> - {model} - - ))} - - + )} - - {!isAutoMode && Add Models} - {isAutoMode && ( - - Don't see your model? Add it manually: - - )} - - setManualModelInput(e.target.value)} - placeholder="e.g., llama3, gpt-4o, claude-sonnet-4-20250514" - onKeyDown={(e) => { - if (e.key === "Enter") { - e.preventDefault() - handleAddManualModel() - } - }} - /> - - Add - - - {isAutoMode && - selectedModels.filter((m) => !fetchedModels.includes(m)).length > - 0 && ( + {!isAutoMode && ( + + Add Models + + setManualModelInput(e.target.value)} + placeholder="e.g., llama3, gpt-4o, claude-sonnet-4-20250514" + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault() + handleAddManualModel() + } + }} + /> + + Add + + + {manualModels.length > 0 && ( - {selectedModels - .filter((m) => !fetchedModels.includes(m)) - .map((model) => ( - ( + + {model} + handleRemoveManualModel(model)} > - {model} - handleToggleModel(model)} - > - - - - ))} + + + + ))} )} - {!isAutoMode && manualModels.length > 0 && ( - - {manualModels.map((model) => ( - - {model} - handleRemoveManualModel(model)} - > - - - - ))} - - )} - + + )} AI Assistant uses tools to gather information about QuestDB and your database. Make sure to select the models that support tool calling. diff --git a/src/components/SetupAIAssistant/ReasoningSection.tsx b/src/components/SetupAIAssistant/ReasoningSection.tsx new file mode 100644 index 000000000..16adf22db --- /dev/null +++ b/src/components/SetupAIAssistant/ReasoningSection.tsx @@ -0,0 +1,77 @@ +import React from "react" +import styled from "styled-components" +import { SelectMenu } from "../SelectMenu" + +export type ReasoningEffortLevel = "default" | "high" + +type Option = { + level: ReasoningEffortLevel + label: string +} + +const OPTIONS: Option[] = [ + { level: "default", label: "Default" }, + { level: "high", label: "High" }, +] + +const Field = styled.div` + display: flex; + flex-direction: column; + gap: 1.6rem; + font-size: 1.1rem; + width: 100%; +` + +const RichTitle = styled.span` + font-size: 1.6rem; + font-weight: 600; + color: ${({ theme }) => theme.color.contentPrimary}; +` + +type Props = { + value: ReasoningEffortLevel + onChange: (next: ReasoningEffortLevel) => void + disabled?: boolean +} + +export const ReasoningSection: React.FC = ({ + value, + onChange, + disabled = false, +}) => { + const current = OPTIONS.find((o) => o.level === value) ?? OPTIONS[0] + + return ( + + Reasoning + + + + + onChange(level as ReasoningEffortLevel)} + > + {OPTIONS.map((opt) => ( + + {opt.label} + + ))} + + + + + + ) +} diff --git a/src/components/SetupAIAssistant/SettingsModal.tsx b/src/components/SetupAIAssistant/SettingsModal.tsx index bb6623644..812c20767 100644 --- a/src/components/SetupAIAssistant/SettingsModal.tsx +++ b/src/components/SetupAIAssistant/SettingsModal.tsx @@ -1,37 +1,40 @@ -import React, { useState, useCallback, useMemo, useRef } from "react" +import React, { useState, useCallback, useEffect, useMemo, useRef } from "react" import styled, { useTheme } from "styled-components" import * as RadixDialog from "@radix-ui/react-dialog" import { Dialog } from "../Dialog" import { Box } from "../Box" import { Input } from "../Input" -import { Switch } from "../Switch" import { Text } from "../Text" import { Button } from "../Button" import { IconButton } from "../IconButton" import { TabButton } from "../TabButton" import { TextButton } from "../TextButton" import { useLocalStorage } from "../../providers/LocalStorageProvider" -import { testApiKey } from "../../utils/ai/aiAssistant" import { StoreKey } from "../../utils/localStorage/types" import { toast } from "../Toast" import { Edit } from "../icons" import { TrashIcon, PlugsIcon, PlusIcon, XIcon } from "@phosphor-icons/react" import { OpenAIIcon } from "./OpenAIIcon" import { AnthropicIcon } from "./AnthropicIcon" -import { BrainIcon } from "./BrainIcon" import { LoadingSpinner } from "../LoadingSpinner" import { Overlay } from "../Overlay" import { getAllProviders, getAllModelOptions, getApiKey, - makeCustomModelValue, + makeModelValue, + stripModelNamespace, + formatModelLabel, + buildProviderSettings, BUILTIN_PROVIDERS, type ModelOption, type ProviderId, + type ProviderModel, getNextModel, getProviderName, + getModelListingErrorMessage, } from "../../utils/ai" +import { createProvider } from "../../utils/ai/registry" import type { AiAssistantSettings, CustomProviderDefinition, @@ -45,6 +48,9 @@ import { trackEvent } from "../../modules/ConsoleEventTracker" import { ConsoleEvent } from "../../modules/ConsoleEventTracker/events" import { CustomProviderModal } from "./CustomProviderModal" import { ManageModelsModal } from "./ManageModelsModal" +import type { BuiltinModelsResult } from "./ManageModelsModal" +import { ReasoningSection } from "./ReasoningSection" +import type { ReasoningEffortLevel } from "./ReasoningSection" const ModalContent = styled.div` display: flex; @@ -328,13 +334,6 @@ const ModelInfoColumn = styled(Box).attrs({ align-items: flex-start; ` -const ModelInfoRow = styled(Box).attrs({ - gap: "0.8rem", - align: "center", -})` - width: 100%; -` - const ModelDescriptionText = styled(Text)` font-size: 1.1rem; color: ${({ theme }) => theme.color.contentSecondary}; @@ -473,6 +472,32 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { [], ), ) + const [modelLabels, setModelLabels] = useState< + Record> + >(() => + initializeProviderState( + (provider) => + aiAssistantSettings.providers?.[provider]?.modelLabels ?? {}, + {}, + ), + ) + const [utilityModels, setUtilityModels] = useState< + Record + >(() => + initializeProviderState( + (provider) => aiAssistantSettings.providers?.[provider]?.utilityModel, + undefined, + ), + ) + const [reasoningEffort, setReasoningEffort] = useState< + Record + >(() => + initializeProviderState( + (provider) => + aiAssistantSettings.providers?.[provider]?.reasoningEffort ?? "default", + "default", + ), + ) const [permissions, setPermissions] = useState< Record >(() => @@ -515,7 +540,8 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { const inputRef = useRef(null) const [customProviderModalOpen, setCustomProviderModalOpen] = useState(false) - const [manageModelsModalOpen, setManageModelsModalOpen] = useState(false) + const [manageModelsProvider, setManageModelsProvider] = + useState(null) const [localCustomProviders, setLocalCustomProviders] = useState< Record @@ -532,15 +558,45 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { [aiAssistantSettings, localCustomProviders], ) - const handleProviderSelect = useCallback((provider: ProviderId) => { - setSelectedProvider(provider) - setValidationErrors((prev) => ({ ...prev, [provider]: null })) + const handleProviderSelect = useCallback( + (provider: ProviderId) => { + if (provider !== selectedProvider) { + abortValidation(selectedProvider) + } + setSelectedProvider(provider) + setValidationErrors((prev) => ({ ...prev, [provider]: null })) + }, + [selectedProvider], + ) + + const validationTokenRef = useRef>({}) + const [validationListings, setValidationListings] = useState< + Record + >({}) + const mountedRef = useRef(true) + + useEffect(() => { + return () => { + mountedRef.current = false + } }, []) + const abortValidation = (provider: ProviderId) => { + validationTokenRef.current[provider] = + (validationTokenRef.current[provider] ?? 0) + 1 + setValidationState((prev) => + prev[provider] === "validating" ? { ...prev, [provider]: "idle" } : prev, + ) + setValidationListings((prev) => ({ ...prev, [provider]: undefined })) + } + const handleApiKeyChange = useCallback( (provider: ProviderId, value: string) => { + validationTokenRef.current[provider] = + (validationTokenRef.current[provider] ?? 0) + 1 setApiKeys((prev) => ({ ...prev, [provider]: value })) setValidationErrors((prev) => ({ ...prev, [provider]: null })) + setValidationState((prev) => ({ ...prev, [provider]: "idle" })) if (validatedApiKeys[provider]) { setValidatedApiKeys((prev) => ({ ...prev, [provider]: false })) @@ -563,70 +619,56 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { setValidationState((prev) => ({ ...prev, [provider]: "validating" })) setValidationErrors((prev) => ({ ...prev, [provider]: null })) - const providerModels = getModelsForProvider(provider, localSettings) - if (providerModels.length === 0) { - setValidationState((prev) => ({ ...prev, [provider]: "error" })) - setValidationErrors((prev) => ({ - ...prev, - [provider]: "No models available for this provider", - })) - return - } - - const testModel = ( - providerModels.find((m) => m.isTestModel) ?? providerModels[0] - ).value + const token = validationTokenRef.current[provider] ?? 0 + const isStale = () => + !mountedRef.current || + (validationTokenRef.current[provider] ?? 0) !== token + const isBuiltin = !!BUILTIN_PROVIDERS[provider] try { - const result = await testApiKey( - apiKey, - testModel, - provider, - localSettings, - ) - if (!result.valid) { - setValidationState((prev) => ({ ...prev, [provider]: "error" })) + const aiProvider = createProvider(provider, apiKey, localSettings) + const listing = await aiProvider.listModels() + if (isStale()) return + if (isBuiltin) { + setValidationListings((prev) => ({ ...prev, [provider]: listing })) + } + setValidationState((prev) => ({ ...prev, [provider]: "validated" })) + setValidatedApiKeys((prev) => ({ ...prev, [provider]: true })) + setValidationErrors((prev) => ({ ...prev, [provider]: null })) + const storedKey = localSettings.providers?.[provider]?.apiKey + if (isBuiltin && apiKey !== storedKey) { + setEnabledModels((prev) => ({ ...prev, [provider]: [] })) + setModelLabels((prev) => ({ ...prev, [provider]: {} })) + setUtilityModels((prev) => ({ ...prev, [provider]: undefined })) + } + if (isBuiltin) { + setManageModelsProvider(provider) + } + } catch (err) { + if (isStale()) return + const aiProvider = createProvider(provider, apiKey, localSettings) + const classified = aiProvider.classifyError(err, () => {}) + if (!isBuiltin && classified.type !== "invalid_key") { + // Custom endpoints may not implement model listing or use standard + // HTTP statuses. Preserve the existing manual-configuration path. + setValidationState((prev) => ({ ...prev, [provider]: "validated" })) + setValidatedApiKeys((prev) => ({ ...prev, [provider]: true })) setValidationErrors((prev) => ({ ...prev, - [provider]: result.error || "Invalid API key", + [provider]: classified.message, })) - } else { - const defaultModels = getAllModelOptions(localSettings) - .filter((m) => m.defaultEnabled && m.provider === provider) - .map((m) => m.value) - if (defaultModels.length > 0) { - setEnabledModels((prev) => ({ ...prev, [provider]: defaultModels })) - } - setValidationState((prev) => ({ ...prev, [provider]: "validated" })) - setValidatedApiKeys((prev) => ({ ...prev, [provider]: true })) - setValidationErrors((prev) => ({ ...prev, [provider]: null })) + return } - } catch (err) { setValidationState((prev) => ({ ...prev, [provider]: "error" })) - const errorMessage = - err instanceof Error ? err.message : "Failed to validate API key" - setValidationErrors((prev) => ({ ...prev, [provider]: errorMessage })) + setValidatedApiKeys((prev) => ({ ...prev, [provider]: false })) + setValidationErrors((prev) => ({ + ...prev, + [provider]: getModelListingErrorMessage(err, classified), + })) } }, [apiKeys, localSettings], ) - const handleModelToggle = useCallback( - (provider: ProviderId, modelValue: string) => { - void trackEvent(ConsoleEvent.AI_SETTINGS_MODEL_TOGGLE) - setEnabledModels((prev) => { - const current = prev[provider] - const isEnabled = current.includes(modelValue) - return { - ...prev, - [provider]: isEnabled - ? current.filter((m) => m !== modelValue) - : [...current, modelValue], - } - }) - }, - [], - ) - // Emit the legacy schema-access-removed event on grantSchemaAccess → false // so existing dashboards keep working. const handlePermissionsChange = useCallback( @@ -650,13 +692,15 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { const isCustom = !BUILTIN_PROVIDERS[provider] if (validatedApiKeys[provider] || isCustom) { const perms = permissions[provider] - updatedProviders[provider] = { + const labels = modelLabels[provider] + updatedProviders[provider] = buildProviderSettings({ apiKey: apiKeys[provider] ?? "", enabledModels: enabledModels[provider], - grantSchemaAccess: perms.grantSchemaAccess, - read: perms.read, - write: perms.write, - } + permissions: perms, + modelLabels: labels, + utilityModel: utilityModels[provider], + reasoningEffort: reasoningEffort[provider], + }) } else { delete updatedProviders[provider] } @@ -708,6 +752,9 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { localCustomProviders, apiKeys, enabledModels, + modelLabels, + utilityModels, + reasoningEffort, permissions, validatedApiKeys, updateSettings, @@ -720,6 +767,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { const handleRemoveProvider = useCallback( (providerId: ProviderId) => { + abortValidation(providerId) const isCustom = !BUILTIN_PROVIDERS[providerId] void trackEvent(ConsoleEvent.AI_SETTINGS_PROVIDER_REMOVE, { isCustom, @@ -741,6 +789,9 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { setValidationState((prev) => ({ ...prev, [providerId]: "idle" })) setValidationErrors((prev) => ({ ...prev, [providerId]: null })) setEnabledModels((prev) => ({ ...prev, [providerId]: [] })) + setModelLabels((prev) => ({ ...prev, [providerId]: {} })) + setUtilityModels((prev) => ({ ...prev, [providerId]: undefined })) + setReasoningEffort((prev) => ({ ...prev, [providerId]: "default" })) setIsInputFocused((prev) => ({ ...prev, [providerId]: false })) // Switch to first remaining active provider @@ -762,7 +813,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { const handleCustomProviderSave = useCallback( (providerId: string, definition: CustomProviderDefinition) => { const newEnabledModels = definition.models.map((m) => - makeCustomModelValue(providerId, m), + makeModelValue(providerId, m), ) setLocalCustomProviders((prev) => ({ @@ -819,7 +870,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { const handleManageModelsSave = useCallback( (providerId: string, definition: CustomProviderDefinition) => { const newModelValues = definition.models.map((m) => - makeCustomModelValue(providerId, m), + makeModelValue(providerId, m), ) // Update local custom providers — only override models and contextWindow, @@ -836,7 +887,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { // Determine which models are truly new (not in the previous model list) const oldModelValues = ( localCustomProviders[providerId]?.models || [] - ).map((m) => makeCustomModelValue(providerId, m)) + ).map((m) => makeModelValue(providerId, m)) const trulyNew = newModelValues.filter((m) => !oldModelValues.includes(m)) // Local state: respect unsaved checkbox toggles, add truly new as enabled @@ -887,6 +938,103 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { [aiAssistantSettings, enabledModels, localCustomProviders, updateSettings], ) + const builtinModelsSavedRef = useRef(false) + + const handleBuiltinModelsOpenChange = useCallback( + (nextOpen: boolean) => { + if (nextOpen) return + const provider = manageModelsProvider + setManageModelsProvider(null) + const cancelled = !builtinModelsSavedRef.current + builtinModelsSavedRef.current = false + if (!provider) return + setValidationListings((prev) => ({ ...prev, [provider]: undefined })) + if (!cancelled || (enabledModels[provider]?.length ?? 0) > 0) return + abortValidation(provider) + const stored = aiAssistantSettings.providers?.[provider] + if (stored?.enabledModels?.length) { + setApiKeys((prev) => ({ ...prev, [provider]: stored.apiKey })) + setEnabledModels((prev) => ({ + ...prev, + [provider]: stored.enabledModels, + })) + setModelLabels((prev) => ({ + ...prev, + [provider]: stored.modelLabels ?? {}, + })) + setUtilityModels((prev) => ({ + ...prev, + [provider]: stored.utilityModel, + })) + return + } + setValidatedApiKeys((prev) => ({ ...prev, [provider]: false })) + setValidationState((prev) => ({ ...prev, [provider]: "idle" })) + }, + [aiAssistantSettings, enabledModels, manageModelsProvider], + ) + + const handleBuiltinModelsSave = useCallback( + (providerId: string, result: BuiltinModelsResult) => { + builtinModelsSavedRef.current = true + const modelValues = result.enabledModels.map((model) => + makeModelValue(providerId, model), + ) + setEnabledModels((prev) => ({ + ...prev, + [providerId]: modelValues, + })) + setModelLabels((prev) => ({ ...prev, [providerId]: result.modelLabels })) + setUtilityModels((prev) => ({ + ...prev, + [providerId]: result.utilityModel, + })) + + const storedProvider = aiAssistantSettings.providers?.[providerId] + // The nested dialog commits its API key and model preferences only. + // Permission and reasoning edits remain drafts until Save Settings. + const persistedPermissions: Permissions = storedProvider + ? { + grantSchemaAccess: storedProvider.grantSchemaAccess, + read: storedProvider.read === true, + write: storedProvider.write === true, + } + : { grantSchemaAccess: true, read: false, write: false } + const updatedSettings: AiAssistantSettings = { + ...aiAssistantSettings, + providers: { + ...aiAssistantSettings.providers, + [providerId]: buildProviderSettings({ + apiKey: apiKeys[providerId] ?? "", + enabledModels: modelValues, + permissions: persistedPermissions, + modelLabels: result.modelLabels, + utilityModel: result.utilityModel, + reasoningEffort: storedProvider?.reasoningEffort ?? "default", + }), + }, + } + const persistedEnabledModels = Object.fromEntries( + Object.entries(updatedSettings.providers).map( + ([provider, providerSettings]) => [ + provider, + providerSettings?.enabledModels ?? [], + ], + ), + ) + updatedSettings.selectedModel = + getNextModel( + updatedSettings.selectedModel, + persistedEnabledModels, + updatedSettings, + ) || undefined + + updateSettings(StoreKey.AI_ASSISTANT_SETTINGS, updatedSettings) + toast.success("Model preferences updated") + }, + [aiAssistantSettings, apiKeys, updateSettings], + ) + const currentProviderValidated = validatedApiKeys[selectedProvider] const currentProviderApiKey = apiKeys[selectedProvider] const currentProviderValidationState = validationState[selectedProvider] @@ -907,6 +1055,12 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { [enabledModels, selectedProvider], ) + const labelForModel = (provider: ProviderId, value: string) => { + const modelId = stripModelNamespace(value, provider) + if (!BUILTIN_PROVIDERS[provider]) return modelId + return modelLabels[provider]?.[modelId] ?? formatModelLabel(modelId) + } + const allProviders = useMemo( () => getAllProviders(localSettings), [localSettings], @@ -929,7 +1083,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { return ( <> @@ -1152,56 +1306,39 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { align="center" style={{ width: "100%" }} > - Enable Models - {isCustomProvider && - (currentProviderValidated || - modelsForProvider.length > 0) && ( - setManageModelsModalOpen(true)} - > - Manage models - - )} + Models + {(currentProviderValidated || + (isCustomProvider && + modelsForProvider.length > 0)) && ( + + setManageModelsProvider(selectedProvider) + } + > + Manage models + + )} - {currentProviderValidated || - (isCustomProvider && modelsForProvider.length > 0) ? ( + {enabledModelsForProvider.length > 0 ? ( - {modelsForProvider.map((model) => { - const isEnabled = enabledModelsForProvider.includes( - model.value, + {enabledModelsForProvider.map((value) => { + const label = labelForModel(selectedProvider, value) + const modelId = stripModelNamespace( + value, + selectedProvider, ) return ( - + - {model.label} - {model.isSlow && ( - - - - Due to advanced reasoning & thinking - capabilities, responses using this model - can be slow. - - + {label} + {!isCustomProvider && label !== modelId && ( + + {modelId} + )} - - handleModelToggle( - selectedProvider, - model.value, - ) - } - /> ) })} @@ -1209,14 +1346,28 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { ) : ( - When you've entered and validated your API key, - you'll be able to select and enable available - models. + {currentProviderValidated + ? "No models enabled yet. Use “Manage models” to enable the models you want." + : "When you’ve entered and validated your API key, you’ll be able to enable models via “Manage models”."} )} + {selectedProvider === "openai" && ( + + + setReasoningEffort((prev) => ({ + ...prev, + [selectedProvider]: next, + })) + } + disabled={!currentProviderValidated} + /> + + )} { )} /> )} - {manageModelsModalOpen && - isCustomProvider && - localCustomProviders[selectedProvider] && ( + {manageModelsProvider && + !BUILTIN_PROVIDERS[manageModelsProvider] && + localCustomProviders[manageModelsProvider] && ( { + if (!nextOpen) setManageModelsProvider(null) + }} + providerId={manageModelsProvider} + definition={localCustomProviders[manageModelsProvider]} onSave={handleManageModelsSave} /> )} + {manageModelsProvider && BUILTIN_PROVIDERS[manageModelsProvider] && ( + stripModelNamespace(model, manageModelsProvider), + )} + initialListing={validationListings[manageModelsProvider]} + onSave={handleBuiltinModelsSave} + /> + )} ) } diff --git a/src/components/ValidationNotice/index.tsx b/src/components/ValidationNotice/index.tsx new file mode 100644 index 000000000..b4f70f6f0 --- /dev/null +++ b/src/components/ValidationNotice/index.tsx @@ -0,0 +1,37 @@ +import React, { ReactNode } from "react" +import styled from "styled-components" +import { WarningIcon } from "@phosphor-icons/react" + +const Notice = styled.div` + display: flex; + flex-shrink: 0; + align-items: center; + gap: 0.6rem; + width: 100%; + padding: 0.6rem 2.4rem; + background: ${({ theme }) => theme.color.statusDangerSurface}; + border-top: 0.1rem solid ${({ theme }) => theme.color.interactionNeutral}; + color: ${({ theme }) => theme.color.contentPrimary}; + font-size: 1.3rem; + line-height: 1.3; + + > svg { + flex-shrink: 0; + color: ${({ theme }) => theme.color.statusDanger}; + } +` + +type ValidationNoticeProps = { + children: ReactNode + dataHook: string +} + +export const ValidationNotice = ({ + children, + dataHook, +}: ValidationNoticeProps) => ( + + + {children} + +) diff --git a/src/providers/LocalStorageProvider/index.tsx b/src/providers/LocalStorageProvider/index.tsx index 36fab08b0..8c66a5be2 100644 --- a/src/providers/LocalStorageProvider/index.tsx +++ b/src/providers/LocalStorageProvider/index.tsx @@ -32,6 +32,7 @@ import React, { useRef, } from "react" import { getValue, setValue } from "../../utils/localStorage" +import { migrateLocalStorage } from "../../utils/localStorage/migrate" import { StoreKey } from "../../utils/localStorage/types" import { parseInteger, @@ -41,6 +42,7 @@ import { } from "./utils" import type { MaxColumnWidth } from "../../components/ResultGrid/types" import { + AI_MODEL_VALUE_FORMAT, AiAssistantSettings, LocalConfig, SettingsType, @@ -49,9 +51,10 @@ import { NotebookOnboarding, RunWithSelectionMode, } from "./types" -import { reconcileSettings } from "../../utils/ai/settings" +import { onReasoningUnsupported } from "../../utils/ai/reasoningFallback" export const DEFAULT_AI_ASSISTANT_SETTINGS: AiAssistantSettings = { + modelValueFormat: AI_MODEL_VALUE_FORMAT, providers: {}, } @@ -157,24 +160,21 @@ type ContextProps = { } const getAiAssistantSettings = (): AiAssistantSettings => { - const stored = getValue(StoreKey.AI_ASSISTANT_SETTINGS) - if (stored) { - try { + try { + const stored = getValue(StoreKey.AI_ASSISTANT_SETTINGS) + if (stored) { const parsed = JSON.parse(stored) as AiAssistantSettings - const reconciled = reconcileSettings({ + return { + modelValueFormat: parsed.modelValueFormat, selectedModel: parsed.selectedModel, providers: parsed.providers || {}, ...(parsed.customProviders && { customProviders: parsed.customProviders, }), - }) - if (JSON.stringify(reconciled) !== stored) { - setValue(StoreKey.AI_ASSISTANT_SETTINGS, JSON.stringify(reconciled)) } - return reconciled - } catch (e) { - return defaultConfig.aiAssistantSettings } + } catch { + return defaultConfig.aiAssistantSettings } return defaultConfig.aiAssistantSettings } @@ -277,7 +277,10 @@ export const LocalStorageProvider = ({ useState(getLeftPanelState()) const [aiAssistantSettings, setAiAssistantSettings] = - useState(getAiAssistantSettings()) + useState(() => { + if (!migrateLocalStorage()) return defaultConfig.aiAssistantSettings + return getAiAssistantSettings() + }) const [aiChatPanelWidth, setAiChatPanelWidth] = useState( parseInteger( @@ -381,6 +384,23 @@ export const LocalStorageProvider = ({ [refreshSettings], ) + useEffect( + () => + onReasoningUnsupported((providerId) => { + const settings = getAiAssistantSettings() + const providerSettings = settings.providers[providerId] + if (providerSettings?.reasoningEffort !== "high") return + updateSettings(StoreKey.AI_ASSISTANT_SETTINGS, { + ...settings, + providers: { + ...settings.providers, + [providerId]: { ...providerSettings, reasoningEffort: "default" }, + }, + }) + }), + [updateSettings], + ) + const value = useMemo( () => ({ editorCol, diff --git a/src/providers/LocalStorageProvider/types.ts b/src/providers/LocalStorageProvider/types.ts index aad36d921..cfe0e39d5 100644 --- a/src/providers/LocalStorageProvider/types.ts +++ b/src/providers/LocalStorageProvider/types.ts @@ -7,6 +7,9 @@ export type ProviderSettings = { // Optional for back-compat; missing fields default to denied. read?: boolean write?: boolean + modelLabels?: Record + utilityModel?: string + reasoningEffort?: "default" | "high" } export type CustomProviderDefinition = { @@ -21,7 +24,11 @@ export type CustomProviderDefinition = { write?: boolean } +export const AI_MODEL_VALUE_FORMAT = 2 as const + export type AiAssistantSettings = { + /** Version 2 stores globally referenced models as `providerId:modelId`. */ + modelValueFormat?: typeof AI_MODEL_VALUE_FORMAT selectedModel?: string providers: Partial> customProviders?: Record diff --git a/src/scenes/Footer/MCPBridgeStatus/PermissionsSection.tsx b/src/scenes/Footer/MCPBridgeStatus/PermissionsSection.tsx index 3c4bb728a..d4194472e 100644 --- a/src/scenes/Footer/MCPBridgeStatus/PermissionsSection.tsx +++ b/src/scenes/Footer/MCPBridgeStatus/PermissionsSection.tsx @@ -62,8 +62,12 @@ const FieldLabel = styled.span` font-weight: 600; ` +const RichField = styled(Field)` + gap: 1.6rem; +` + const RichTitle = styled.span` - font-size: 1.8rem; + font-size: 1.6rem; font-weight: 600; color: ${({ theme }) => theme.color.contentPrimary}; ` @@ -123,13 +127,13 @@ export const PermissionsSection: React.FC = ({ if (variant === "rich") { return ( - + Permissions {trigger} {content} - + ) } diff --git a/src/utils/ai/aiAssistant.ts b/src/utils/ai/aiAssistant.ts index 157640543..89f6ee398 100644 --- a/src/utils/ai/aiAssistant.ts +++ b/src/utils/ai/aiAssistant.ts @@ -350,16 +350,6 @@ const tryWithRetries = async ( } } -export const testApiKey = async ( - apiKey: string, - model: string, - providerId: ProviderId, - settings?: AiAssistantSettings, -): Promise<{ valid: boolean; error?: string }> => { - const provider = createProvider(providerId, apiKey, settings) - return provider.testConnection({ apiKey, model }) -} - export const generateChatTitle = async ({ firstUserMessage, settings, diff --git a/src/utils/ai/anthropicProvider.namespace.test.ts b/src/utils/ai/anthropicProvider.namespace.test.ts new file mode 100644 index 000000000..aec776900 --- /dev/null +++ b/src/utils/ai/anthropicProvider.namespace.test.ts @@ -0,0 +1,85 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" +import { createAnthropicProvider } from "./anthropicProvider" + +const { createMock, streamMock, countTokensMock } = vi.hoisted(() => ({ + createMock: vi.fn(), + streamMock: vi.fn(), + countTokensMock: vi.fn(), +})) + +vi.mock("@anthropic-ai/sdk", () => ({ + default: class MockAnthropic { + messages = { + create: createMock, + stream: streamMock, + countTokens: countTokensMock, + } + models = { list: vi.fn() } + }, +})) + +const message = { + id: "msg_1", + type: "message", + role: "assistant", + model: "claude-test", + content: [{ type: "text", text: "done" }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, +} + +const summaryStream = () => + (async function* () { + await Promise.resolve() + yield { + type: "content_block_delta", + delta: { type: "text_delta", text: "summary" }, + } + })() + +const capturedModels = (mock: { + mock: { calls: unknown[][] } +}): Array => + mock.mock.calls.map( + (call) => (call[0] as { model?: string } | undefined)?.model, + ) + +describe("anthropic provider model namespaces", () => { + beforeEach(() => { + createMock.mockReset().mockResolvedValue(message) + streamMock.mockReset().mockImplementation(summaryStream) + countTokensMock.mockReset().mockResolvedValue({ input_tokens: 7 }) + }) + + it("sends raw model ids at every Anthropic request boundary", async () => { + const provider = createAnthropicProvider("sk-test") + const model = "anthropic:claude-test" + + await provider.executeFlow({ + model, + config: { + systemInstructions: "system", + initialUserContent: "hello", + }, + modelToolsClient: {} as never, + tools: [], + setStatus: () => {}, + }) + await provider.generateTitle({ model, prompt: "title" }) + await provider.generateSummary({ + model, + systemPrompt: "system", + userMessage: "summarize", + }) + await provider.countTokens({ + model, + systemPrompt: "system", + messages: [{ role: "user", content: "hello" }], + }) + + expect(capturedModels(createMock)).toEqual(["claude-test", "claude-test"]) + expect(capturedModels(streamMock)).toEqual(["claude-test"]) + expect(capturedModels(countTokensMock)).toEqual(["claude-test"]) + }) +}) diff --git a/src/utils/ai/anthropicProvider.test.ts b/src/utils/ai/anthropicProvider.test.ts index b6f0a523b..7bba9c0fc 100644 --- a/src/utils/ai/anthropicProvider.test.ts +++ b/src/utils/ai/anthropicProvider.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest" -import { toNativeMessages } from "./anthropicProvider" +import Anthropic from "@anthropic-ai/sdk" +import { createAnthropicProvider, toNativeMessages } from "./anthropicProvider" import type { Message, ToolCall } from "./types" const toolCall = (over: Partial = {}): ToolCall => ({ @@ -10,6 +11,30 @@ const toolCall = (over: Partial = {}): ToolCall => ({ ...over, }) +describe("anthropicProvider.classifyError", () => { + it("reports a 429 as a rate or usage limit", () => { + const provider = createAnthropicProvider("test-key") + const error = new Anthropic.RateLimitError( + 429, + { + type: "error", + error: { + type: "rate_limit_error", + message: "Rate limit reached", + }, + }, + "Rate limit reached", + new Headers(), + ) + + expect(provider.classifyError(error, () => {})).toMatchObject({ + type: "rate_limit", + message: + "The provider's rate or usage limit was reached. Check your provider account limits or try again later.", + }) + }) +}) + describe("anthropicProvider.toNativeMessages", () => { it("converts a plain user message to a user role param", () => { // Given a single user message diff --git a/src/utils/ai/anthropicProvider.ts b/src/utils/ai/anthropicProvider.ts index 5a2731f55..9ca7e4d61 100644 --- a/src/utils/ai/anthropicProvider.ts +++ b/src/utils/ai/anthropicProvider.ts @@ -8,12 +8,13 @@ import type { StreamingCallback, TokenUsage, } from "./aiAssistant" -import { getModelProps } from "./settings" +import { stripModelNamespace } from "./settings" import type { ProviderId } from "./settings" import { type AIProvider, type ExecuteFlowParams, type FlowResult, + type ProviderModel, type ToolDefinition, type Message, } from "./types" @@ -129,10 +130,6 @@ function toAnthropicTools(tools: ToolDefinition[]): AnthropicTool[] { })) } -function toAnthropicModel(model: string): string { - return getModelProps(model).model -} - async function createAnthropicMessage( anthropic: Anthropic, params: Omit & { @@ -397,7 +394,6 @@ async function handleToolCalls( system: systemPrompt, ...(!isLastRound && { tools }), messages: updatedHistory, - temperature: 0.3, } const followUpMessage = streaming @@ -500,14 +496,13 @@ export function createAnthropicProvider( const toolContext: ToolExecutionContext = incomingToolContext ?? {} - const resolvedModel = toAnthropicModel(model) + const resolvedModel = stripModelNamespace(model, providerId) const messageParams: Parameters[1] = { model: resolvedModel, system: systemPrompt, tools: anthropicTools, messages: initialMessages, - temperature: 0.3, } const message = streaming @@ -584,10 +579,9 @@ export function createAnthropicProvider( async generateTitle({ model, prompt }) { try { const message = await createAnthropicMessage(anthropic, { - model: toAnthropicModel(model), + model: stripModelNamespace(model, providerId), messages: [{ role: "user", content: prompt }], max_tokens: 100, - temperature: 0.3, }) const textBlock = message.content.find((block) => block.type === "text") @@ -611,7 +605,7 @@ export function createAnthropicProvider( let text = "" const stream = anthropic.messages.stream( { - ...getModelProps(model), + model: stripModelNamespace(model, providerId), max_tokens: 64_000, messages: [{ role: "user", content: userMessage }], system: systemPrompt, @@ -629,54 +623,6 @@ export function createAnthropicProvider( return text }, - async testConnection({ apiKey: testApiKey, model }) { - try { - const testClient = new Anthropic({ - apiKey: testApiKey, - dangerouslyAllowBrowser: true, - ...(options?.baseURL ? { baseURL: options.baseURL } : {}), - ...(isCustom - ? { - fetch: createHeaderFilteredFetch(ANTHROPIC_ALLOWED_HEADERS), - } - : {}), - }) - - await createAnthropicMessage(testClient, { - model: toAnthropicModel(model), - messages: [{ role: "user", content: "ping" }], - max_tokens: 16, - }) - return { valid: true } - } catch (error: unknown) { - if (error instanceof MaxTokensError || error instanceof RefusalError) { - return { valid: true } - } - if (error instanceof Anthropic.AuthenticationError) { - return { valid: false, error: "Invalid API key" } - } - if (error instanceof Anthropic.RateLimitError) { - return { valid: true } - } - const status = - (error as { status?: number })?.status || - (error as { error?: { status?: number } })?.error?.status - if (status === 401) { - return { valid: false, error: "Invalid API key" } - } - if (status === 429) { - return { valid: true } - } - return { - valid: false, - error: - error instanceof Error - ? error.message - : "Failed to validate API key", - } - } - }, - async countTokens({ messages, systemPrompt, model }) { // Custom providers (non-default baseURL) use chars/3.5 estimation // because the actual tokenizer is unknown and most custom endpoints @@ -690,19 +636,23 @@ export function createAnthropicProvider( const nativeMessages = toNativeMessages(messages) const response = await anthropic.messages.countTokens({ - model: toAnthropicModel(model), + model: stripModelNamespace(model, providerId), system: systemPrompt, messages: nativeMessages, }) return response.input_tokens }, - async listModels(): Promise { - const models: string[] = [] + async listModels(): Promise { + const models: ProviderModel[] = [] for await (const model of anthropic.models.list()) { - models.push(model.id) + models.push({ + id: model.id, + label: model.display_name, + created: Math.floor(Date.parse(model.created_at) / 1000), + }) } - return models.sort((a, b) => a.localeCompare(b)) + return models.sort((a, b) => a.id.localeCompare(b.id)) }, classifyError( @@ -767,7 +717,8 @@ export function createAnthropicProvider( if (error instanceof Anthropic.RateLimitError) { return { type: "rate_limit", - message: "Rate limit exceeded. Please try again later.", + message: + "The provider's rate or usage limit was reached. Check your provider account limits or try again later.", details: error.message, } } diff --git a/src/utils/ai/contextCompaction.ts b/src/utils/ai/contextCompaction.ts index 31761ad56..b9bf202ff 100644 --- a/src/utils/ai/contextCompaction.ts +++ b/src/utils/ai/contextCompaction.ts @@ -1,5 +1,5 @@ import type { ConversationMessage } from "../../providers/AIConversationProvider/types" -import { getTestModel } from "./index" +import { getSelectedModel } from "./index" import type { AIProvider, Message } from "./index" import { getMessageTextLength } from "./shared" import type { AiAssistantSettings } from "../../providers/LocalStorageProvider/types" @@ -89,9 +89,9 @@ async function generateSummary( settings?: AiAssistantSettings, abortSignal?: AbortSignal, ): Promise { - const testModelValue = getTestModel(aiProvider.id, settings) - if (!testModelValue) { - throw new Error("No test model found for provider") + const summaryModel = settings ? getSelectedModel(settings) : null + if (!summaryModel) { + throw new Error("No model selected for summarization") } const conversationText = middleMessages @@ -123,7 +123,7 @@ async function generateSummary( const userMessage = `Please summarize the following conversation:\n\n${conversationText}` return aiProvider.generateSummary({ - model: testModelValue, + model: summaryModel, systemPrompt: SUMMARIZATION_PROMPT, userMessage, abortSignal, diff --git a/src/utils/ai/executeAIFlow.ts b/src/utils/ai/executeAIFlow.ts index 26c69f188..e16f90204 100644 --- a/src/utils/ai/executeAIFlow.ts +++ b/src/utils/ai/executeAIFlow.ts @@ -33,7 +33,7 @@ import { type StreamingCallback, } from "./aiAssistant" import { getExplainSchemaPrompt, getHealthIssuePrompt } from "./index" -import { providerForModel, getTestModel, getAllModelOptions } from "./index" +import { providerForModel, getUtilityModel, getAllModelOptions } from "./index" import type { AiAssistantSettings } from "../../providers/LocalStorageProvider/types" import { eventBus } from "../../modules/EventBus" import { EventType } from "../../modules/EventBus/types" @@ -415,14 +415,14 @@ async function generateChatTitleIfNeeded( ) if (!provider) return - const testModelValue = getTestModel(provider, config.aiAssistantSettings) - if (!testModelValue) return + const utilityModel = getUtilityModel(provider, config.aiAssistantSettings) + if (!utilityModel) return try { const title = await generateChatTitle({ firstUserMessage: userMessageContent, settings: { - model: testModelValue, + model: utilityModel, provider, apiKey: config.settings.apiKey, aiAssistantSettings: config.aiAssistantSettings, diff --git a/src/utils/ai/index.ts b/src/utils/ai/index.ts index f587f1556..8af428fc2 100644 --- a/src/utils/ai/index.ts +++ b/src/utils/ai/index.ts @@ -24,21 +24,22 @@ export { } from "./prompts" export type { HealthIssuePromptData } from "./prompts" export { - MODEL_OPTIONS, BUILTIN_PROVIDERS, + buildListingMetadata, providerForModel, - getModelProps, getProviderName, getAllProviders, getAllModelOptions, getAllEnabledModels, + getModelLabel, getSelectedModel, getNextModel, - getTestModel, + getUtilityModel, getProviderContextWindow, getApiKey, - makeCustomModelValue, - parseModelValue, + buildProviderSettings, + makeModelValue, + stripModelNamespace, isAiAssistantConfigured, canUseAiAssistant, hasSchemaAccess, @@ -51,3 +52,12 @@ export type { ModelOption, CustomProviderDefinition, } from "./settings" +export { + filterOpenAiChatModels, + formatModelLabel, + resolveUtilityModel, + sortModelsNewestFirst, + UTILITY_MODEL_TIERS, +} from "./modelCatalog" +export type { ProviderModel } from "./modelCatalog" +export { getModelListingErrorMessage } from "./modelListingError" diff --git a/src/utils/ai/modelCatalog.test.ts b/src/utils/ai/modelCatalog.test.ts new file mode 100644 index 000000000..41bbea2d5 --- /dev/null +++ b/src/utils/ai/modelCatalog.test.ts @@ -0,0 +1,163 @@ +import { describe, it, expect } from "vitest" +import { + filterOpenAiChatModels, + formatModelLabel, + resolveUtilityModel, + stripDateSuffix, + UTILITY_MODEL_TIERS, +} from "./modelCatalog" +import type { ProviderModel } from "./modelCatalog" + +const model = (id: string, created?: number): ProviderModel => ({ id, created }) + +const AUG_2025 = Date.UTC(2025, 7, 7) / 1000 +const JUL_2025 = Date.UTC(2025, 6, 1) / 1000 + +describe("stripDateSuffix", () => { + it("strips Anthropic and OpenAI date suffixes", () => { + expect(stripDateSuffix("claude-sonnet-4-5-20250929")).toBe( + "claude-sonnet-4-5", + ) + expect(stripDateSuffix("gpt-4.1-2025-04-14")).toBe("gpt-4.1") + expect(stripDateSuffix("gpt-4-0613")).toBe("gpt-4") + }) + + it("keeps non-date suffixes", () => { + expect(stripDateSuffix("gpt-3.5-turbo-16k")).toBe("gpt-3.5-turbo-16k") + expect(stripDateSuffix("claude-sonnet-4-5")).toBe("claude-sonnet-4-5") + }) +}) + +describe("formatModelLabel", () => { + it("derives labels from OpenAI ids", () => { + expect(formatModelLabel("gpt-5-mini")).toBe("GPT 5 Mini") + expect(formatModelLabel("gpt-5.4")).toBe("GPT 5.4") + expect(formatModelLabel("gpt-5.6-luna")).toBe("GPT 5.6 Luna") + expect(formatModelLabel("gpt-4o")).toBe("GPT 4o") + expect(formatModelLabel("o4-mini")).toBe("o4 Mini") + }) + + it("formats version numbers without provider-specific separators", () => { + expect(formatModelLabel("gpt-6")).toBe("GPT 6") + expect(formatModelLabel("gpt-6-7")).toBe("GPT 6.7") + expect(formatModelLabel("gpt-5.6-7")).toBe("GPT 5.6.7") + expect(formatModelLabel("gpt-5.6-something")).toBe("GPT 5.6 Something") + expect(formatModelLabel("something-1.2")).toBe("Something 1.2") + expect(formatModelLabel("claude-sonnet-4-5")).toBe("Claude Sonnet 4.5") + expect(formatModelLabel("claude-sonnet-5")).toBe("Claude Sonnet 5") + expect(formatModelLabel("claude-sonnet-6-7")).toBe("Claude Sonnet 6.7") + expect(formatModelLabel("claude-sonnet-6.7-something")).toBe( + "Claude Sonnet 6.7 Something", + ) + expect(formatModelLabel("claude-sonnet-6-7-something")).toBe( + "Claude Sonnet 6.7 Something", + ) + }) + + it("preserves date suffixes", () => { + expect(formatModelLabel("gpt-5.4-20250815")).toBe("GPT 5.4 (20250815)") + expect(formatModelLabel("claude-opus-4-5-20251101")).toBe( + "Claude Opus 4.5 (20251101)", + ) + expect(formatModelLabel("gpt-5.4-nano-2026-03-17")).toBe( + "GPT 5.4 Nano (2026-03-17)", + ) + }) +}) + +describe("filterOpenAiChatModels", () => { + it("drops known non-chat models but keeps dated snapshots", () => { + // Given a listing with chat models, noise, and dated snapshots + const listing = [ + model("gpt-5.4", AUG_2025 + 300), + model("gpt-5.4-2026-03-05", AUG_2025 + 300), + model("text-embedding-3-small", AUG_2025 + 1), + model("whisper-1", AUG_2025 + 1), + model("gpt-4o-mini-tts", AUG_2025 + 1), + model("gpt-5-chat-latest", AUG_2025 + 200), + model("gpt-5.3-codex", AUG_2025 + 250), + model("gpt-5.4-pro", AUG_2025 + 250), + model("sora-2", AUG_2025 + 250), + model("davinci-002", AUG_2025 + 1), + ] + // When the filter runs + const kept = filterOpenAiChatModels(listing).map((m) => m.id) + // Then both the plain chat model and its dated snapshot remain + expect(kept).toEqual(["gpt-5.4", "gpt-5.4-2026-03-05"]) + }) + + it("hides generations older than gpt-5 by default", () => { + // Given chat models from before and after the gpt-5 launch + const listing = [ + model("gpt-5", AUG_2025), + model("gpt-4.1", JUL_2025), + model("gpt-3.5-turbo", JUL_2025 - 1_000_000), + ] + // When the filter runs + const kept = filterOpenAiChatModels(listing).map((m) => m.id) + // Then only the current generation stays in the default view + expect(kept).toEqual(["gpt-5"]) + }) + + it("keeps a brand-new generation without any code change", () => { + const kept = filterOpenAiChatModels([model("gpt-6", AUG_2025 + 500)]).map( + (m) => m.id, + ) + expect(kept).toEqual(["gpt-6"]) + }) + + it("sorts newest first", () => { + const kept = filterOpenAiChatModels([ + model("gpt-5", AUG_2025 + 100), + model("gpt-5.4", AUG_2025 + 300), + model("gpt-5.2", AUG_2025 + 200), + ]).map((m) => m.id) + expect(kept).toEqual(["gpt-5.4", "gpt-5.2", "gpt-5"]) + }) +}) + +describe("resolveUtilityModel", () => { + it("picks the newest model of the highest-priority tier", () => { + // Given luna and nano models where nano is newer + const listing = [ + model("gpt-5.6-luna", 300), + model("gpt-5.7-nano", 400), + model("gpt-5.4-mini", 200), + ] + // When resolving with OpenAI tiers + const utility = resolveUtilityModel(listing, UTILITY_MODEL_TIERS.openai) + // Then priority beats recency + expect(utility).toBe("gpt-5.6-luna") + }) + + it("falls through to lower tiers when the top tier is absent", () => { + const listing = [model("gpt-5.4-mini", 200), model("gpt-5.4-nano", 200)] + expect(resolveUtilityModel(listing, UTILITY_MODEL_TIERS.openai)).toBe( + "gpt-5.4-nano", + ) + }) + + it("dedupes dated variants per alias and returns the listed id verbatim", () => { + const listing = [ + model("claude-haiku-4-5-20251001", 100), + model("claude-haiku-4-6", 200), + model("claude-sonnet-5", 300), + ] + expect(resolveUtilityModel(listing, UTILITY_MODEL_TIERS.anthropic)).toBe( + "claude-haiku-4-6", + ) + }) + + it("keeps a dated winner's exact listed id", () => { + const listing = [model("claude-haiku-4-5-20251001", 100)] + expect(resolveUtilityModel(listing, UTILITY_MODEL_TIERS.anthropic)).toBe( + "claude-haiku-4-5-20251001", + ) + }) + + it("returns null when no tier matches", () => { + expect( + resolveUtilityModel([model("gpt-5.4", 100)], UTILITY_MODEL_TIERS.openai), + ).toBeNull() + }) +}) diff --git a/src/utils/ai/modelCatalog.ts b/src/utils/ai/modelCatalog.ts new file mode 100644 index 000000000..2a30fb4e8 --- /dev/null +++ b/src/utils/ai/modelCatalog.ts @@ -0,0 +1,116 @@ +export type ProviderModel = { + id: string + label?: string + created?: number +} + +const DATE_SUFFIX = /-(\d{8}|\d{4}-\d{2}-\d{2}|\d{4})$/ + +// Models released since GPT-5 are eligible for the picker; older models remain +// available through manual entry. +const GPT5_LAUNCH_START = Date.UTC(2025, 7, 1) / 1000 + +const OPENAI_NON_CHAT_TOKENS = [ + "embedding", + "tts", + "whisper", + "audio", + "realtime", + "image", + "dall-e", + "sora", + "transcribe", + "transcription", + "moderation", + "search", + "codex", + "computer-use", + "chat-latest", + // Research-tier models: minutes of reasoning with no streamed output or + // summaries — they read as unresponsive in an interactive chat. + "-pro", + "chatgpt", + "babbage", + "davinci", + "instruct", +] + +export const stripDateSuffix = (id: string): string => + id.replace(DATE_SUFFIX, "") + +const isNumericToken = (token: string): boolean => /^[\d.]+$/.test(token) + +export const formatModelLabel = (id: string): string => { + const dateSuffix = id.match(DATE_SUFFIX) + const baseId = dateSuffix ? id.slice(0, -dateSuffix[0].length) : id + const tokens = baseId.split("-") + const parts: string[] = [] + for (const token of tokens) { + if (token.toLowerCase() === "gpt") { + parts.push("GPT") + continue + } + const previous = parts[parts.length - 1] + if (isNumericToken(token)) { + if (previous && isNumericToken(previous)) { + parts[parts.length - 1] = `${previous}.${token}` + } else { + parts.push(token) + } + continue + } + if (/\d/.test(token)) { + parts.push(token) + continue + } + parts.push(token.charAt(0).toUpperCase() + token.slice(1)) + } + const label = parts.join(" ") || id + return dateSuffix ? `${label} (${dateSuffix[1]})` : label +} + +export const sortModelsNewestFirst = ( + models: ProviderModel[], +): ProviderModel[] => + [...models].sort( + (a, b) => (b.created ?? 0) - (a.created ?? 0) || a.id.localeCompare(b.id), + ) + +const isOpenAiNonChatModel = (id: string): boolean => + OPENAI_NON_CHAT_TOKENS.some((token) => id.toLowerCase().includes(token)) + +export const filterOpenAiChatModels = ( + models: ProviderModel[], +): ProviderModel[] => + sortModelsNewestFirst( + models.filter( + (m) => + !isOpenAiNonChatModel(m.id) && (m.created ?? 0) >= GPT5_LAUNCH_START, + ), + ) + +export const UTILITY_MODEL_TIERS: Record<"anthropic" | "openai", string[]> = { + anthropic: ["haiku", "sonnet"], + openai: ["luna", "nano", "mini"], +} + +export const resolveUtilityModel = ( + models: ProviderModel[], + tiers: string[], +): string | null => { + const byAlias = new Map() + for (const model of models) { + const alias = stripDateSuffix(model.id) + const existing = byAlias.get(alias) + if (!existing || (model.created ?? 0) > (existing.created ?? 0)) { + byAlias.set(alias, model) + } + } + for (const tier of tiers) { + const matches = sortModelsNewestFirst( + [...byAlias.values()].filter((m) => m.id.includes(tier)), + ) + if (matches.length > 0) return matches[0].id + } + return null +} diff --git a/src/utils/ai/modelListingError.test.ts b/src/utils/ai/modelListingError.test.ts new file mode 100644 index 000000000..587e973ec --- /dev/null +++ b/src/utils/ai/modelListingError.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest" +import { getModelListingErrorMessage } from "./modelListingError" +import type { AiAssistantAPIError } from "./aiAssistant" + +const classified = ( + type: AiAssistantAPIError["type"] = "unknown", +): AiAssistantAPIError => ({ type, message: "Provider error" }) + +describe("getModelListingErrorMessage", () => { + it.each([ + [401, "Invalid API key"], + [403, "This API key does not have permission to list models"], + [404, "This provider does not support model listing"], + [405, "This provider does not support model listing"], + [429, "The provider rate limit was reached"], + [500, "The provider is temporarily unavailable"], + [503, "The provider is temporarily unavailable"], + ])("maps HTTP %s to a specific message", (status, expected) => { + expect(getModelListingErrorMessage({ status }, classified())).toContain( + expected, + ) + }) + + it("describes connection failures without claiming validation succeeded", () => { + expect( + getModelListingErrorMessage( + new Error("fetch failed"), + classified("network"), + ), + ).toBe( + "Could not reach the provider. Check its URL and your network connection.", + ) + }) + + it("preserves the provider message for other failures", () => { + expect( + getModelListingErrorMessage( + { status: 422 }, + { type: "unknown", message: "Unsupported request" }, + ), + ).toBe("Unsupported request") + }) +}) diff --git a/src/utils/ai/modelListingError.ts b/src/utils/ai/modelListingError.ts new file mode 100644 index 000000000..4fe53d750 --- /dev/null +++ b/src/utils/ai/modelListingError.ts @@ -0,0 +1,36 @@ +import type { AiAssistantAPIError } from "./aiAssistant" + +const getHttpStatus = (error: unknown): number | null => { + if (typeof error !== "object" || error === null || !("status" in error)) { + return null + } + const status = (error as { status?: unknown }).status + return typeof status === "number" ? status : null +} + +export const getModelListingErrorMessage = ( + error: unknown, + classified: AiAssistantAPIError, +): string => { + const status = getHttpStatus(error) + + if (status === 401 || classified.type === "invalid_key") { + return "Invalid API key" + } + if (status === 403) { + return "This API key does not have permission to list models" + } + if (status === 404 || status === 405) { + return "This provider does not support model listing. Configure its models manually." + } + if (status === 429 || classified.type === "rate_limit") { + return "The provider rate limit was reached. Please try again later." + } + if (status !== null && status >= 500) { + return "The provider is temporarily unavailable. Please try again later." + } + if (classified.type === "network") { + return "Could not reach the provider. Check its URL and your network connection." + } + return classified.message +} diff --git a/src/utils/ai/openaiChatCompletionsProvider.ts b/src/utils/ai/openaiChatCompletionsProvider.ts index b4db2775e..ec0de616b 100644 --- a/src/utils/ai/openaiChatCompletionsProvider.ts +++ b/src/utils/ai/openaiChatCompletionsProvider.ts @@ -9,12 +9,13 @@ import type { StreamingCallback, TokenUsage, } from "./aiAssistant" -import { getModelProps } from "./settings" +import { stripModelNamespace } from "./settings" import type { ProviderId } from "./settings" import { type AIProvider, type ExecuteFlowParams, type FlowResult, + type ProviderModel, type ToolDefinition, type Message, } from "./types" @@ -350,23 +351,14 @@ async function executeRequest( } } -function toChatCompletionsAPIProps(model: string): { - model: string - reasoning_effort?: OpenAI.ReasoningEffort -} { - const props = getModelProps(model) - return { - model: props.model, - ...(props.reasoningEffort - ? { reasoning_effort: props.reasoningEffort as OpenAI.ReasoningEffort } - : {}), - } -} - export function createOpenAIChatCompletionsProvider( apiKey: string, providerId: ProviderId = "openai", - options?: { baseURL?: string; contextWindow?: number; isCustom?: boolean }, + options?: { + baseURL?: string + contextWindow?: number + isCustom?: boolean + }, ): AIProvider { const isCustom = options?.isCustom ?? false const openai = new OpenAI({ @@ -421,7 +413,7 @@ export function createOpenAIChatCompletionsProvider( const toolContext: ToolExecutionContext = incomingToolContext ?? {} const baseParams = { - ...toChatCompletionsAPIProps(model), + model: stripModelNamespace(model, providerId), tools: openaiTools, } @@ -563,7 +555,7 @@ export function createOpenAIChatCompletionsProvider( async generateTitle({ model, prompt }) { try { const response = await openai.chat.completions.create({ - model: toChatCompletionsAPIProps(model).model, + model: stripModelNamespace(model, providerId), messages: [{ role: "user", content: prompt }], ...(isCustom ? {} : { max_completion_tokens: 100 }), }) @@ -585,16 +577,20 @@ export function createOpenAIChatCompletionsProvider( abortSignal?: AbortSignal }) { let text = "" + const summaryParams = { + model: stripModelNamespace(model, providerId), + messages: [ + { role: "system" as const, content: systemPrompt }, + { role: "user" as const, content: userMessage }, + ], + stream: true as const, + } + const requestOptions = abortSignal + ? ([{ signal: abortSignal }] as const) + : ([] as const) const stream = await openai.chat.completions.create( - { - ...toChatCompletionsAPIProps(model), - messages: [ - { role: "system", content: systemPrompt }, - { role: "user", content: userMessage }, - ], - stream: true, - }, - ...(abortSignal ? [{ signal: abortSignal }] : ([] as const)), + summaryParams, + ...requestOptions, ) for await (const chunk of stream) { const delta = chunk.choices[0]?.delta?.content @@ -605,43 +601,6 @@ export function createOpenAIChatCompletionsProvider( return text }, - async testConnection({ apiKey: testApiKey, model }) { - try { - const testClient = new OpenAI({ - apiKey: testApiKey, - dangerouslyAllowBrowser: true, - ...(options?.baseURL ? { baseURL: options.baseURL } : {}), - ...(isCustom - ? { - fetch: createHeaderFilteredFetch(OPENAI_ALLOWED_HEADERS), - } - : {}), - }) - await testClient.chat.completions.create({ - model: getModelProps(model).model, - messages: [{ role: "user", content: "ping" }], - }) - return { valid: true } - } catch (error: unknown) { - const status = - (error as { status?: number })?.status || - (error as { error?: { status?: number } })?.error?.status - if (status === 401) { - return { valid: false, error: "Invalid API key" } - } - if (status === 429) { - return { valid: true } - } - return { - valid: false, - error: - error instanceof Error - ? error.message - : "Failed to validate API key", - } - } - }, - async countTokens({ messages, systemPrompt }) { // Custom providers (non-default baseURL) use chars/3.5 estimation if (options?.baseURL) { @@ -655,12 +614,17 @@ export function createOpenAIChatCompletionsProvider( return countTokensFromNativePayload(systemPrompt, nativeMessages) }, - async listModels(): Promise { - const models: string[] = [] + async listModels(): Promise { + const models: ProviderModel[] = [] for await (const model of openai.models.list()) { - models.push(model.id) + const name = (model as { name?: unknown }).name + models.push({ + id: model.id, + created: model.created, + ...(typeof name === "string" ? { label: name } : {}), + }) } - return models.sort((a, b) => a.localeCompare(b)) + return models.sort((a, b) => a.id.localeCompare(b.id)) }, classifyError( diff --git a/src/utils/ai/openaiProvider.fallback.test.ts b/src/utils/ai/openaiProvider.fallback.test.ts new file mode 100644 index 000000000..cc6db3f5b --- /dev/null +++ b/src/utils/ai/openaiProvider.fallback.test.ts @@ -0,0 +1,166 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest" +import { createOpenAIProvider } from "./openaiProvider" +import { onReasoningUnsupported } from "./reasoningFallback" + +const { createMock, toastInfoMock } = vi.hoisted(() => ({ + createMock: vi.fn(), + toastInfoMock: vi.fn(), +})) + +vi.mock("openai", async (importOriginal) => { + const actual = await importOriginal() + class MockOpenAI { + static APIError = actual.default.APIError + static APIUserAbortError = actual.default.APIUserAbortError + responses = { create: createMock } + } + return { default: MockOpenAI } +}) + +vi.mock("../../components/Toast", () => ({ + toast: { info: toastInfoMock }, +})) + +const reasoningRejection = async () => { + const OpenAI = (await import("openai")).default + const message = + "Unsupported parameter: 'reasoning.effort' is not supported with this model." + return new OpenAI.APIError( + 400, + { message, param: "reasoning.effort", type: "invalid_request_error" }, + message, + undefined, + ) +} + +const requestBodies = () => + createMock.mock.calls.map( + (call: unknown[]) => call[0] as Record, + ) + +const textStream = (text: string) => + (async function* () { + await Promise.resolve() + yield { type: "response.output_text.delta", delta: text } + })() + +describe("openai reasoning fallback", () => { + let unsupportedProviders: string[] + let unregister: () => void + + beforeEach(() => { + createMock.mockReset() + toastInfoMock.mockReset() + unsupportedProviders = [] + unregister = onReasoningUnsupported((providerId) => + unsupportedProviders.push(providerId), + ) + }) + + afterEach(() => { + unregister() + }) + + it("sends high effort, then strips reasoning and retries once on a rejection", async () => { + // Given a model that rejects the reasoning parameter + createMock + .mockRejectedValueOnce(await reasoningRejection()) + .mockReturnValueOnce(textStream("summary")) + const provider = createOpenAIProvider("sk-test", "openai", { + reasoning: { effort: "high" }, + }) + + // When generating a summary + const text = await provider.generateSummary({ + model: "openai:gpt-4o", + systemPrompt: "sys", + userMessage: "user", + }) + + // Then the first request carried reasoning and the retry dropped it + expect(text).toBe("summary") + expect(createMock).toHaveBeenCalledTimes(2) + expect(requestBodies()[0].reasoning).toEqual({ + effort: "high", + summary: "auto", + }) + expect(requestBodies()[0].model).toBe("gpt-4o") + expect(requestBodies()[1].model).toBe("gpt-4o") + expect("reasoning" in requestBodies()[1]).toBe(false) + + // And the downgrade was surfaced and reported exactly once + expect(toastInfoMock).toHaveBeenCalledWith( + "Reasoning preference changed to Default", + ) + expect(unsupportedProviders).toEqual(["openai"]) + }) + + it("stops sending reasoning on later requests after a rejection", async () => { + createMock + .mockRejectedValueOnce(await reasoningRejection()) + .mockReturnValue(textStream("again")) + const provider = createOpenAIProvider("sk-test", "openai", { + reasoning: { effort: "high" }, + }) + await provider.generateSummary({ + model: "gpt-4o", + systemPrompt: "sys", + userMessage: "user", + }) + + // When a second request runs on the same provider instance + await provider.generateSummary({ + model: "gpt-4o", + systemPrompt: "sys", + userMessage: "user", + }) + + // Then it goes out once, without reasoning, and nothing is re-reported + expect(createMock).toHaveBeenCalledTimes(3) + expect("reasoning" in requestBodies()[2]).toBe(false) + expect(toastInfoMock).toHaveBeenCalledTimes(1) + expect(unsupportedProviders).toEqual(["openai"]) + }) + + it("rethrows unrelated 400s without retrying", async () => { + const OpenAI = (await import("openai")).default + const message = "Item of type 'reasoning' was provided without its pair." + createMock.mockRejectedValueOnce( + new OpenAI.APIError( + 400, + { message, param: null, type: "invalid_request_error" }, + message, + undefined, + ), + ) + const provider = createOpenAIProvider("sk-test", "openai", { + reasoning: { effort: "high" }, + }) + + await expect( + provider.generateSummary({ + model: "gpt-4o", + systemPrompt: "sys", + userMessage: "user", + }), + ).rejects.toThrow() + + expect(createMock).toHaveBeenCalledTimes(1) + expect(toastInfoMock).not.toHaveBeenCalled() + expect(unsupportedProviders).toEqual([]) + }) + + it("sends no reasoning at all when effort is default", async () => { + createMock.mockReturnValueOnce(textStream("plain")) + const provider = createOpenAIProvider("sk-test", "openai") + + await provider.generateSummary({ + model: "gpt-5.4", + systemPrompt: "sys", + userMessage: "user", + }) + + expect(createMock).toHaveBeenCalledTimes(1) + expect("reasoning" in requestBodies()[0]).toBe(false) + }) +}) diff --git a/src/utils/ai/openaiProvider.ts b/src/utils/ai/openaiProvider.ts index 14dac9d68..4164f1792 100644 --- a/src/utils/ai/openaiProvider.ts +++ b/src/utils/ai/openaiProvider.ts @@ -5,12 +5,15 @@ import type { StatusCallback, StreamingCallback, } from "./aiAssistant" -import { getModelProps } from "./settings" +import { stripModelNamespace } from "./settings" import type { ProviderId } from "./settings" +import { reportReasoningUnsupported } from "./reasoningFallback" +import { toast } from "../../components/Toast" import { type AIProvider, type ExecuteFlowParams, type FlowResult, + type ProviderModel, type ToolDefinition, type Message, } from "./types" @@ -28,6 +31,7 @@ import { classifyOpenAIError, countTokensFromNativePayload, isOpenAINonRetryableError, + isReasoningRejection, } from "./openaiShared" import { createHeaderFilteredFetch, @@ -130,7 +134,6 @@ async function createOpenAIResponseStreaming( ...params, stream: true, store: false, - include: ["reasoning.encrypted_content"], } as OpenAI.Responses.ResponseCreateParamsStreaming, { signal: abortSignal }, ) @@ -255,28 +258,26 @@ function getOpenAIText(response: OpenAI.Responses.Response): { return { type: "text", message: "" } } -function toResponsesAPIProps(model: string): { - model: string - reasoning?: OpenAI.Reasoning -} { - const props = getModelProps(model) - return { - model: props.model, - ...(props.reasoningEffort - ? { - reasoning: { - effort: props.reasoningEffort as OpenAI.ReasoningEffort, - summary: "auto", - }, - } - : {}), - } +function stripReasoning( + params: OpenAI.Responses.ResponseCreateParamsNonStreaming, +): OpenAI.Responses.ResponseCreateParamsNonStreaming { + const { + reasoning: _reasoning, + include: _include, + ...withoutReasoning + } = params + return withoutReasoning } export function createOpenAIProvider( apiKey: string, providerId: ProviderId = "openai", - options?: { baseURL?: string; contextWindow?: number; isCustom?: boolean }, + options?: { + baseURL?: string + contextWindow?: number + isCustom?: boolean + reasoning?: { effort: "high" } + }, ): AIProvider { const isCustom = options?.isCustom ?? false const openai = new OpenAI({ @@ -292,6 +293,61 @@ export function createOpenAIProvider( const contextWindow = options?.contextWindow ?? 400_000 + let reasoningUnsupported = false + + const markReasoningUnsupported = () => { + if (reasoningUnsupported) return + reasoningUnsupported = true + toast.info("Reasoning preference changed to Default") + reportReasoningUnsupported(providerId) + } + + const toRequestProps = ( + model: string, + ): { model: string; reasoning?: OpenAI.Reasoning } => { + return { + model: stripModelNamespace(model, providerId), + ...(options?.reasoning && !reasoningUnsupported + ? { + reasoning: { + effort: options.reasoning.effort, + summary: "auto" as const, + }, + } + : {}), + } + } + + const createResponseWithReasoningFallback = async ( + params: OpenAI.Responses.ResponseCreateParamsNonStreaming, + streaming?: StreamingCallback, + abortSignal?: AbortSignal, + ): Promise => { + const run = async (p: OpenAI.Responses.ResponseCreateParamsNonStreaming) => + streaming + ? ( + await createOpenAIResponseStreaming( + openai, + p, + streaming, + abortSignal, + ) + ).response + : openai.responses.create(p) + const effectiveParams = reasoningUnsupported + ? stripReasoning(params) + : params + try { + return await run(effectiveParams) + } catch (error) { + if (effectiveParams.reasoning && isReasoningRejection(error)) { + markReasoningUnsupported() + return run(stripReasoning(params)) + } + throw error + } + } + return { id: providerId, contextWindow, @@ -329,7 +385,7 @@ export function createOpenAIProvider( const toolContext: ToolExecutionContext = incomingToolContext ?? {} const requestParams = { - ...toResponsesAPIProps(model), + ...toRequestProps(model), instructions: config.systemInstructions, input, tools: openaiTools, @@ -337,17 +393,11 @@ export function createOpenAIProvider( include: ["reasoning.encrypted_content"], } as OpenAI.Responses.ResponseCreateParamsNonStreaming - const streamResult = streaming - ? await createOpenAIResponseStreaming( - openai, - requestParams, - streaming, - abortSignal, - ) - : { - response: await openai.responses.create(requestParams), - } - let lastResponse = streamResult.response + let lastResponse = await createResponseWithReasoningFallback( + requestParams, + streaming, + abortSignal, + ) input = [...input, ...lastResponse.output] totalInputTokens += lastResponse.usage?.input_tokens ?? 0 @@ -437,7 +487,7 @@ export function createOpenAIProvider( streaming?.onResponseStart?.() const loopRequestParams = { - ...toResponsesAPIProps(model), + ...toRequestProps(model), instructions: config.systemInstructions, input, ...(!isLastRound && { tools: openaiTools }), @@ -445,17 +495,11 @@ export function createOpenAIProvider( include: ["reasoning.encrypted_content"], } as OpenAI.Responses.ResponseCreateParamsNonStreaming - const loopResult = streaming - ? await createOpenAIResponseStreaming( - openai, - loopRequestParams, - streaming, - abortSignal, - ) - : { - response: await openai.responses.create(loopRequestParams), - } - lastResponse = loopResult.response + lastResponse = await createResponseWithReasoningFallback( + loopRequestParams, + streaming, + abortSignal, + ) input = [...input, ...lastResponse.output] totalInputTokens += lastResponse.usage?.input_tokens ?? 0 @@ -502,7 +546,7 @@ export function createOpenAIProvider( async generateTitle({ model, prompt }) { try { const response = await openai.responses.create({ - model: toResponsesAPIProps(model).model, + model: stripModelNamespace(model, providerId), input: [{ role: "user", content: prompt }], max_output_tokens: 100, }) @@ -524,15 +568,29 @@ export function createOpenAIProvider( abortSignal?: AbortSignal }) { let text = "" - const stream = await openai.responses.create( - { - ...toResponsesAPIProps(model), - instructions: systemPrompt, - input: userMessage, - stream: true, - }, - ...(abortSignal ? [{ signal: abortSignal }] : ([] as const)), - ) + const summaryParams = { + ...toRequestProps(model), + instructions: systemPrompt, + input: userMessage, + stream: true as const, + } + const requestOptions = abortSignal + ? ([{ signal: abortSignal }] as const) + : ([] as const) + let stream + try { + stream = await openai.responses.create(summaryParams, ...requestOptions) + } catch (error) { + if (!summaryParams.reasoning || !isReasoningRejection(error)) { + throw error + } + markReasoningUnsupported() + const { reasoning: _reasoning, ...withoutReasoning } = summaryParams + stream = await openai.responses.create( + withoutReasoning, + ...requestOptions, + ) + } for await (const event of stream) { if (event.type === "response.output_text.delta" && "delta" in event) { text += event.delta @@ -541,43 +599,6 @@ export function createOpenAIProvider( return text }, - async testConnection({ apiKey: testApiKey, model }) { - try { - const testClient = new OpenAI({ - apiKey: testApiKey, - dangerouslyAllowBrowser: true, - ...(options?.baseURL ? { baseURL: options.baseURL } : {}), - ...(isCustom - ? { - fetch: createHeaderFilteredFetch(OPENAI_ALLOWED_HEADERS), - } - : {}), - }) - await testClient.responses.create({ - model: getModelProps(model).model, - input: [{ role: "user", content: "ping" }], - }) - return { valid: true } - } catch (error: unknown) { - const status = - (error as { status?: number })?.status || - (error as { error?: { status?: number } })?.error?.status - if (status === 401) { - return { valid: false, error: "Invalid API key" } - } - if (status === 429) { - return { valid: true } - } - return { - valid: false, - error: - error instanceof Error - ? error.message - : "Failed to validate API key", - } - } - }, - async countTokens({ messages, systemPrompt }) { // Custom providers (non-default baseURL) use chars/3.5 estimation if (options?.baseURL) { @@ -591,12 +612,17 @@ export function createOpenAIProvider( return countTokensFromNativePayload(systemPrompt, nativeInput) }, - async listModels(): Promise { - const models: string[] = [] + async listModels(): Promise { + const models: ProviderModel[] = [] for await (const model of openai.models.list()) { - models.push(model.id) + const name = (model as { name?: unknown }).name + models.push({ + id: model.id, + created: model.created, + ...(typeof name === "string" ? { label: name } : {}), + }) } - return models.sort((a, b) => a.localeCompare(b)) + return models.sort((a, b) => a.id.localeCompare(b.id)) }, classifyError( diff --git a/src/utils/ai/openaiShared.test.ts b/src/utils/ai/openaiShared.test.ts new file mode 100644 index 000000000..6ad765b4c --- /dev/null +++ b/src/utils/ai/openaiShared.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect } from "vitest" +import OpenAI from "openai" +import { classifyOpenAIError, isReasoningRejection } from "./openaiShared" + +const apiError = (status: number, param: string | null, message: string) => + new OpenAI.APIError( + status, + { message, param, type: "invalid_request_error" }, + message, + undefined, + ) + +describe("isReasoningRejection", () => { + it("matches a 400 whose param names reasoning", () => { + // Given the Responses API rejects reasoning on a non-reasoning model + const responsesError = apiError( + 400, + "reasoning.effort", + "Unsupported parameter: 'reasoning.effort' is not supported with this model.", + ) + const chatError = apiError( + 400, + "reasoning_effort", + "Unsupported parameter: 'reasoning_effort' is not supported with this model.", + ) + + // Then both wire formats are recognized + expect(isReasoningRejection(responsesError)).toBe(true) + expect(isReasoningRejection(chatError)).toBe(true) + }) + + it("ignores a 400 that only mentions reasoning in its message", () => { + // Given an unrelated 400 whose message contains the word "reasoning" + const error = apiError( + 400, + null, + "Item of type 'reasoning' was provided without its required following item.", + ) + + // Then it is not treated as a reasoning rejection + expect(isReasoningRejection(error)).toBe(false) + }) + + it("ignores non-400 statuses and non-API errors", () => { + expect( + isReasoningRejection(apiError(429, "reasoning.effort", "rate limited")), + ).toBe(false) + expect(isReasoningRejection(new Error("reasoning failed"))).toBe(false) + }) +}) + +describe("classifyOpenAIError", () => { + it("reports a 429 as a rate or usage limit", () => { + const error = new OpenAI.RateLimitError( + 429, + { + message: "Rate limit reached", + type: "rate_limit_error", + code: "rate_limit_exceeded", + }, + "Rate limit reached", + new Headers(), + ) + + expect(classifyOpenAIError(error, () => {})).toMatchObject({ + type: "rate_limit", + message: + "The provider's rate or usage limit was reached. Check your provider account limits or try again later.", + }) + }) +}) diff --git a/src/utils/ai/openaiShared.ts b/src/utils/ai/openaiShared.ts index e6209e2bd..7d5a9e8f8 100644 --- a/src/utils/ai/openaiShared.ts +++ b/src/utils/ai/openaiShared.ts @@ -3,6 +3,11 @@ import type { Tiktoken, TiktokenBPE } from "js-tiktoken/lite" import type { StatusCallback, AiAssistantAPIError } from "./aiAssistant" import { StreamingError, RefusalError, MaxTokensError } from "./shared" +export function isReasoningRejection(error: unknown): boolean { + if (!(error instanceof OpenAI.APIError) || error.status !== 400) return false + return typeof error.param === "string" && error.param.startsWith("reasoning") +} + let tiktokenEncoder: Tiktoken | null = null export async function countTokensFromNativePayload( @@ -88,7 +93,8 @@ export function classifyOpenAIError( if (error instanceof OpenAI.RateLimitError) { return { type: "rate_limit", - message: "Rate limit exceeded. Please try again later.", + message: + "The provider's rate or usage limit was reached. Check your provider account limits or try again later.", details: error.message, } } diff --git a/src/utils/ai/reasoningFallback.ts b/src/utils/ai/reasoningFallback.ts new file mode 100644 index 000000000..ec25d3413 --- /dev/null +++ b/src/utils/ai/reasoningFallback.ts @@ -0,0 +1,16 @@ +type ReasoningUnsupportedHandler = (providerId: string) => void + +let handler: ReasoningUnsupportedHandler | null = null + +export const onReasoningUnsupported = ( + nextHandler: ReasoningUnsupportedHandler, +): (() => void) => { + handler = nextHandler + return () => { + if (handler === nextHandler) handler = null + } +} + +export const reportReasoningUnsupported = (providerId: string): void => { + handler?.(providerId) +} diff --git a/src/utils/ai/registry.ts b/src/utils/ai/registry.ts index 947b42e37..2415d8f3e 100644 --- a/src/utils/ai/registry.ts +++ b/src/utils/ai/registry.ts @@ -10,6 +10,17 @@ type ProviderOptions = { baseURL?: string contextWindow?: number isCustom?: boolean + reasoning?: { effort: "high" } +} + +const reasoningOptions = ( + providerId: ProviderId, + settings?: AiAssistantSettings, +): Pick => { + if (BUILTIN_PROVIDERS[providerId]?.type !== "openai") return {} + return settings?.providers?.[providerId]?.reasoningEffort === "high" + ? { reasoning: { effort: "high" } } + : {} } export function createProvider( @@ -20,7 +31,12 @@ export function createProvider( // Check built-in providers first const builtin = BUILTIN_PROVIDERS[providerId] if (builtin) { - return createProviderByType(builtin.type, providerId, apiKey) + return createProviderByType( + builtin.type, + providerId, + apiKey, + reasoningOptions(providerId, settings), + ) } // Check custom providers diff --git a/src/utils/ai/settings.test.ts b/src/utils/ai/settings.test.ts index b12d75713..ab0eb77ba 100644 --- a/src/utils/ai/settings.test.ts +++ b/src/utils/ai/settings.test.ts @@ -1,13 +1,22 @@ -import { describe, it, expect, afterEach, beforeAll } from "vitest" +import { describe, it, expect } from "vitest" import { - reconcileSettings, getSelectedModel, getAiPermissions, - MODEL_OPTIONS, + getAllModelOptions, + getNextModel, + getUtilityModel, + providerForModel, + buildListingMetadata, + makeModelValue, + parseModelValue, + stripModelNamespace, } from "./settings" -import type { ModelOption } from "./settings" +import { migrateLocalStorage } from "../localStorage/migrate" -import type { AiAssistantSettings } from "../../providers/LocalStorageProvider/types" +import { + AI_MODEL_VALUE_FORMAT, + type AiAssistantSettings, +} from "../../providers/LocalStorageProvider/types" const makeSettings = ( overrides: Partial = {}, @@ -16,83 +25,290 @@ const makeSettings = ( ...overrides, }) -describe("reconcileSettings", () => { - it("removes stale model IDs from enabledModels", () => { +const migrateSettings = ( + settings: AiAssistantSettings, +): AiAssistantSettings => { + let stored = JSON.stringify(settings) + migrateLocalStorage({ + getItem: () => stored, + setItem: (_key, value) => { + stored = value + }, + }) + return JSON.parse(stored) as AiAssistantSettings +} + +describe("migrateLocalStorage", () => { + it("isolates local-storage read and write failures", () => { + expect( + migrateLocalStorage({ + getItem: () => { + throw new Error("storage unavailable") + }, + setItem: () => undefined, + }), + ).toBe(false) + + expect( + migrateLocalStorage({ + getItem: () => + JSON.stringify({ + providers: {}, + }), + setItem: () => { + throw new Error("storage quota exceeded") + }, + }), + ).toBe(false) + }) + + it.each([AI_MODEL_VALUE_FORMAT, 3])( + "does not touch settings carrying format version %s", + (modelValueFormat) => { + const stored = JSON.stringify({ + modelValueFormat, + selectedModel: "openai:openai:foo", + providers: { + openai: { + apiKey: "sk-test", + enabledModels: ["openai:openai:foo"], + grantSchemaAccess: false, + }, + }, + futureField: "preserved byte-for-byte", + }) + let writes = 0 + + migrateLocalStorage({ + getItem: () => stored, + setItem: () => { + writes += 1 + }, + }) + + expect(writes).toBe(0) + }, + ) + + it("migrates built-in model values and utility models to provider-qualified storage", () => { const settings = makeSettings({ + selectedModel: "gpt-5.4", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini", "removed-model", "also-removed"], + enabledModels: ["gpt-5.4", "gpt-5-mini"], + utilityModel: "gpt-5-mini", + modelLabels: { "gpt-5.4": "GPT-5.4" }, grantSchemaAccess: false, }, }, }) - const result = reconcileSettings(settings) - expect(result.providers.openai!.enabledModels).toEqual(["gpt-5-mini"]) + + const result = migrateSettings(settings) + + expect(result.modelValueFormat).toBe(AI_MODEL_VALUE_FORMAT) + expect(result.selectedModel).toBe("openai:gpt-5.4") + expect(result.providers.openai!.enabledModels).toEqual([ + "openai:gpt-5.4", + "openai:gpt-5-mini", + ]) + expect(result.providers.openai!.utilityModel).toBe("openai:gpt-5-mini") + expect(result.providers.openai!.modelLabels).toEqual({ + "gpt-5.4": "GPT-5.4", + }) }) - it("does not add defaultEnabled models when user has valid models", () => { + it("preserves a literal provider prefix inside a legacy model id", () => { + const result = migrateSettings( + makeSettings({ + selectedModel: "openai:foo", + providers: { + openai: { + apiKey: "sk-test", + enabledModels: ["openai:foo"], + grantSchemaAccess: false, + }, + }, + }), + ) + + expect(result.selectedModel).toBe("openai:openai:foo") + expect(result.providers.openai!.enabledModels).toEqual([ + "openai:openai:foo", + ]) + expect(stripModelNamespace(result.selectedModel!, "openai")).toBe( + "openai:foo", + ) + }) + + it("keeps built-in model ids it does not recognize", () => { + // Given enabled models that no fixed list knows about const settings = makeSettings({ + providers: { + openai: { + apiKey: "sk-test", + enabledModels: ["gpt-7", "gpt-5-mini"], + grantSchemaAccess: false, + }, + }, + }) + // When storage migrates + const result = migrateSettings(settings) + // Then availability is the picker's job, not the migration's + expect(result.providers.openai!.enabledModels).toEqual([ + "openai:gpt-7", + "openai:gpt-5-mini", + ]) + }) + + it("collapses legacy reasoning variants into plain ids", () => { + const settings = makeSettings({ + providers: { + openai: { + apiKey: "sk-test", + enabledModels: [ + "gpt-5.4@reasoning=high", + "gpt-5.4@reasoning=medium", + "gpt-5-mini", + ], + grantSchemaAccess: false, + }, + }, + }) + const result = migrateSettings(settings) + expect(result.providers.openai!.enabledModels).toEqual([ + "openai:gpt-5.4", + "openai:gpt-5-mini", + ]) + }) + + it("also collapses legacy reasoning variants for built-in Anthropic models", () => { + const settings = makeSettings({ + selectedModel: "claude-sonnet@reasoning=high", providers: { anthropic: { apiKey: "sk-test", - enabledModels: ["claude-sonnet-4-5"], + enabledModels: ["claude-sonnet@reasoning=high"], grantSchemaAccess: false, }, }, }) - const result = reconcileSettings(settings) + const result = migrateSettings(settings) expect(result.providers.anthropic!.enabledModels).toEqual([ - "claude-sonnet-4-5", + "anthropic:claude-sonnet", ]) + expect(result.selectedModel).toBe("anthropic:claude-sonnet") }) - it("leaves enabledModels empty when all previous models were removed", () => { + it("folds a selected high variant into reasoningEffort", () => { + // Given a user who ran the high variant const settings = makeSettings({ + selectedModel: "gpt-5.4@reasoning=high", providers: { - anthropic: { + openai: { apiKey: "sk-test", - enabledModels: ["removed-model-1", "removed-model-2"], + enabledModels: ["gpt-5.4@reasoning=high", "gpt-5.4@reasoning=low"], grantSchemaAccess: false, }, }, }) - const result = reconcileSettings(settings) - expect(result.providers.anthropic!.enabledModels).toEqual([]) + // When storage migrates + const result = migrateSettings(settings) + // Then the provider runs on High and the selection is the plain id + expect(result.providers.openai!.reasoningEffort).toBe("high") + expect(result.selectedModel).toBe("openai:gpt-5.4") }) - it("does not add defaults for unconfigured providers", () => { + it("migrates medium and low variant users to the provider default", () => { const settings = makeSettings({ + selectedModel: "gpt-5.4@reasoning=medium", providers: { - anthropic: { + openai: { apiKey: "sk-test", - enabledModels: ["claude-sonnet-4-5"], + enabledModels: ["gpt-5.4@reasoning=medium", "gpt-5.4@reasoning=low"], + grantSchemaAccess: false, + }, + }, + }) + const result = migrateSettings(settings) + expect(result.providers.openai!.reasoningEffort).toBeUndefined() + expect(result.selectedModel).toBe("openai:gpt-5.4") + }) + + it("removes custom models missing from their provider definition", () => { + const settings = makeSettings({ + customProviders: { + "custom-1": { + type: "openai-chat-completions", + name: "Test", + baseURL: "http://localhost:11434/v1", + contextWindow: 100_000, + models: ["llm-a"], + }, + }, + providers: { + "custom-1": { + apiKey: "", + enabledModels: ["custom-1:llm-a", "custom-1:llm-removed"], grantSchemaAccess: false, }, }, }) - const result = reconcileSettings(settings) - expect(result.providers.openai).toBeUndefined() + const result = migrateSettings(settings) + expect(result.providers["custom-1"]!.enabledModels).toEqual([ + "custom-1:llm-a", + ]) + }) + + it("preserves reasoning-like suffixes in custom provider model ids", () => { + const customModels = [ + "vendor-model@reasoning=high", + "vendor-model@reasoning=medium", + "vendor-model@reasoning=low", + ] + const enabledModels = customModels.map((model) => `custom-1:${model}`) + const settings = makeSettings({ + selectedModel: enabledModels[0], + customProviders: { + "custom-1": { + type: "openai-chat-completions", + name: "Test", + baseURL: "http://localhost:11434/v1", + contextWindow: 100_000, + models: customModels, + }, + }, + providers: { + "custom-1": { + apiKey: "", + enabledModels, + grantSchemaAccess: false, + }, + }, + }) + const result = migrateSettings(settings) + expect(result.providers["custom-1"]!.enabledModels).toEqual(enabledModels) + expect(result.selectedModel).toBe(enabledModels[0]) }) it("is idempotent", () => { const settings = makeSettings({ - selectedModel: "claude-sonnet-4-5", + selectedModel: "gpt-5.4@reasoning=high", providers: { anthropic: { apiKey: "sk-test", - enabledModels: ["claude-sonnet-4-5", "stale-model"], + enabledModels: ["claude-sonnet-4-5"], grantSchemaAccess: true, }, openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini"], + enabledModels: ["gpt-5.4@reasoning=high", "gpt-5-mini"], grantSchemaAccess: false, }, }, }) - const once = reconcileSettings(settings) - const twice = reconcileSettings(once) + const once = migrateSettings(settings) + const twice = migrateSettings(once) expect(twice).toEqual(once) }) @@ -111,13 +327,13 @@ describe("reconcileSettings", () => { string > settingsWithFutureField.futureField = "preserved" - const result = reconcileSettings(settings) + const result = migrateSettings(settings) expect((result as unknown as Record).futureField).toBe( "preserved", ) }) - it("clears selectedModel if not in any enabledModels", () => { + it("repairs selectedModel when it is not enabled anywhere", () => { const settings = makeSettings({ selectedModel: "removed-model", providers: { @@ -128,13 +344,13 @@ describe("reconcileSettings", () => { }, }, }) - const result = reconcileSettings(settings) - expect(result.selectedModel).toEqual("gpt-5-mini") + const result = migrateSettings(settings) + expect(result.selectedModel).toEqual("openai:gpt-5-mini") }) it("preserves selectedModel if it is in enabledModels", () => { const settings = makeSettings({ - selectedModel: "gpt-5-mini", + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "sk-test", @@ -143,13 +359,13 @@ describe("reconcileSettings", () => { }, }, }) - const result = reconcileSettings(settings) - expect(result.selectedModel).toBe("gpt-5-mini") + const result = migrateSettings(settings) + expect(result.selectedModel).toBe("openai:gpt-5-mini") }) it("handles empty providers gracefully", () => { const settings = makeSettings({ providers: {} }) - const result = reconcileSettings(settings) + const result = migrateSettings(settings) expect(result.providers).toEqual({}) }) @@ -158,13 +374,13 @@ describe("reconcileSettings", () => { providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini", "stale-model"], + enabledModels: ["gpt-5-mini", "gpt-5.4@reasoning=high"], grantSchemaAccess: false, }, }, }) const originalModels = [...settings.providers.openai!.enabledModels] - reconcileSettings(settings) + migrateSettings(settings) expect(settings.providers.openai!.enabledModels).toEqual(originalModels) }) }) @@ -172,31 +388,30 @@ describe("reconcileSettings", () => { describe("getSelectedModel", () => { it("returns selectedModel when it is in enabledModels", () => { const settings = makeSettings({ - selectedModel: "gpt-5-mini", + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini", "gpt-5"], + enabledModels: ["openai:gpt-5-mini", "openai:gpt-5"], grantSchemaAccess: false, }, }, }) - expect(getSelectedModel(settings)).toBe("gpt-5-mini") + expect(getSelectedModel(settings)).toBe("openai:gpt-5-mini") }) - it("does not return selectedModel if not in enabledModels", () => { + it("falls back to the first enabled model", () => { const settings = makeSettings({ - selectedModel: "claude-sonnet-4-5", + selectedModel: "anthropic:claude-sonnet-4-5", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini"], + enabledModels: ["openai:gpt-5-mini"], grantSchemaAccess: false, }, }, }) - expect(getSelectedModel(settings)).not.toBe("claude-sonnet-4-5") - expect(getSelectedModel(settings)).toBe("gpt-5-mini") + expect(getSelectedModel(settings)).toBe("openai:gpt-5-mini") }) it("returns null when no models are enabled", () => { @@ -205,180 +420,279 @@ describe("getSelectedModel", () => { }) }) -/** - * Simulates version upgrades by temporarily replacing MODEL_OPTIONS contents. - * Tests verify that user settings from a previous version are handled correctly - * when the app is updated with a different model list. - */ -describe("version compatibility scenarios", () => { - // Snapshot once before any mutation — re-snapshotting per call leaks an empty baseline on throw. - let originalOptions: ModelOption[] - beforeAll(() => { - originalOptions = [...MODEL_OPTIONS] - }) - - function setModelOptions(options: ModelOption[]) { - MODEL_OPTIONS.length = 0 - MODEL_OPTIONS.push(...options) - } - - afterEach(() => { - MODEL_OPTIONS.length = 0 - MODEL_OPTIONS.push(...originalOptions) - }) - - it("upgrade: model removed, selectedModel was that model", () => { - // v1: user had model-A and model-B, selected model-A - setModelOptions([ - { label: "A", value: "model-a", provider: "openai" }, - { label: "B", value: "model-b", provider: "openai" }, - ]) - - const v1Settings = makeSettings({ - selectedModel: "model-a", +describe("getAllModelOptions", () => { + it("keeps identical built-in model ids distinct while showing raw labels", () => { + const settings = makeSettings({ + modelValueFormat: AI_MODEL_VALUE_FORMAT, + selectedModel: "openai:shared-model", providers: { + anthropic: { + apiKey: "sk-ant", + enabledModels: ["anthropic:shared-model"], + grantSchemaAccess: false, + }, openai: { - apiKey: "sk-test", - enabledModels: ["model-a", "model-b"], + apiKey: "sk-openai", + enabledModels: ["openai:shared-model"], grantSchemaAccess: false, }, }, }) - // v2: model-A removed, model-C added - setModelOptions([ - { label: "B", value: "model-b", provider: "openai" }, + const options = getAllModelOptions(settings) + + expect(options).toEqual([ + { + label: "Shared Model", + value: "anthropic:shared-model", + provider: "anthropic", + }, { - label: "C", - value: "model-c", + label: "Shared Model", + value: "openai:shared-model", provider: "openai", - defaultEnabled: true, }, ]) - - const reconciled = reconcileSettings(v1Settings) - expect(reconciled.providers.openai!.enabledModels).toEqual(["model-b"]) - expect(reconciled.selectedModel).toBe("model-b") + expect(new Set(options.map((option) => option.value)).size).toBe(2) + expect(providerForModel(options[1].value, settings)).toBe("openai") }) - it("upgrade: all models removed for a provider", () => { - setModelOptions([{ label: "A", value: "model-a", provider: "openai" }]) - - const v1Settings = makeSettings({ - selectedModel: "model-a", + it("builds options from enabled models with stored labels", () => { + // Given a provider with one stored label and one without + const settings = makeSettings({ providers: { openai: { apiKey: "sk-test", - enabledModels: ["model-a"], + enabledModels: ["openai:gpt-5.4", "openai:gpt-5-mini"], grantSchemaAccess: false, + modelLabels: { "gpt-5.4": "GPT-5.4 (Custom)" }, }, }, }) - - // v2: provider's models completely replaced - setModelOptions([ + // When options build + const options = getAllModelOptions(settings) + // Then the stored label wins and the formatter fills the gap + expect(options).toEqual([ + { + label: "GPT-5.4 (Custom)", + value: "openai:gpt-5.4", + provider: "openai", + }, { - label: "X", - value: "model-x", + label: "GPT 5 Mini", + value: "openai:gpt-5-mini", provider: "openai", - defaultEnabled: true, }, - { label: "Y", value: "model-y", provider: "openai" }, ]) - - const reconciled = reconcileSettings(v1Settings) - // all old models gone, empty list — user must re-enable in settings - expect(reconciled.providers.openai!.enabledModels).toEqual([]) - expect(reconciled.selectedModel).toBeUndefined() - expect(getSelectedModel(reconciled)).toBeNull() }) - it("upgrade: new models added, user keeps their selection", () => { - setModelOptions([ - { label: "A", value: "model-a", provider: "anthropic", default: true }, - { label: "B", value: "model-b", provider: "anthropic" }, + it("includes namespaced custom provider models", () => { + const settings = makeSettings({ + customProviders: { + "custom-1": { + type: "openai-chat-completions", + name: "Test", + baseURL: "http://localhost:11434/v1", + contextWindow: 100_000, + models: ["llm-a"], + }, + }, + }) + expect(getAllModelOptions(settings)).toEqual([ + { label: "llm-a", value: "custom-1:llm-a", provider: "custom-1" }, ]) + }) +}) + +describe("providerForModel", () => { + it("parses built-in and custom provider-qualified values", () => { + const customProviders = { + "custom-1": { + type: "openai-chat-completions" as const, + name: "Test", + baseURL: "http://localhost:11434/v1", + contextWindow: 100_000, + models: ["llm-a"], + }, + } - const v1Settings = makeSettings({ - selectedModel: "model-b", + expect(parseModelValue("openai:gpt-5.4", customProviders)).toEqual({ + providerId: "openai", + rawModel: "gpt-5.4", + }) + expect(parseModelValue("custom-1:llm-a", customProviders)).toEqual({ + providerId: "custom-1", + rawModel: "llm-a", + }) + expect(makeModelValue("openai", "openai:foo")).toBe("openai:openai:foo") + expect(stripModelNamespace("openai:openai:foo", "openai")).toBe( + "openai:foo", + ) + expect(stripModelNamespace("anthropic:shared-model", "openai")).toBe( + "anthropic:shared-model", + ) + }) + + it("finds the built-in provider that enabled the model", () => { + const settings = makeSettings({ providers: { anthropic: { apiKey: "sk-test", - enabledModels: ["model-a", "model-b"], - grantSchemaAccess: true, + enabledModels: ["anthropic:claude-sonnet-5"], + grantSchemaAccess: false, }, }, }) + expect(providerForModel("anthropic:claude-sonnet-5", settings)).toBe( + "anthropic", + ) + expect(providerForModel("gpt-5-mini", settings)).toBeNull() + }) - // v2: model-C added - setModelOptions([ - { label: "A", value: "model-a", provider: "anthropic", default: true }, - { label: "B", value: "model-b", provider: "anthropic" }, - { - label: "C", - value: "model-c", - provider: "anthropic", - defaultEnabled: true, + it("treats a colon prefix as a namespace only when that custom provider exists", () => { + // Given a custom provider named custom-1 + const settings = makeSettings({ + customProviders: { + "custom-1": { + type: "openai-chat-completions", + name: "Test", + baseURL: "http://localhost:11434/v1", + contextWindow: 100_000, + models: ["llm-a"], + }, }, - ]) + }) + // Then only its own prefix resolves as a namespace + expect(providerForModel("custom-1:llm-a", settings)).toBe("custom-1") + expect(providerForModel("custom-1:llm-a")).toBeNull() + }) - const reconciled = reconcileSettings(v1Settings) - // existing models preserved, new model NOT auto-added - expect(reconciled.providers.anthropic!.enabledModels).toEqual([ - "model-a", - "model-b", - ]) - expect(reconciled.selectedModel).toBe("model-b") - expect(getSelectedModel(reconciled)).toBe("model-b") + it("routes colon-containing listed ids to the built-in provider that enabled them", () => { + // Given an OpenAI fine-tune id enabled under the built-in provider + const fineTune = "openai:ft:gpt-4o-mini-2024-07-18:acme::BxK9pQ2r" + const settings = makeSettings({ + providers: { + openai: { + apiKey: "sk-test", + enabledModels: [fineTune], + grantSchemaAccess: false, + }, + }, + }) + // Then the ft: prefix is not mistaken for a custom provider + expect(providerForModel(fineTune, settings)).toBe("openai") }) +}) - it("upgrade: selected model survives but some enabled models removed", () => { - setModelOptions([ - { label: "A", value: "model-a", provider: "openai" }, - { label: "B", value: "model-b", provider: "openai" }, - { label: "C", value: "model-c", provider: "openai" }, - ]) +describe("getNextModel", () => { + it("keeps the current model while it stays enabled", () => { + expect( + getNextModel("openai:gpt-5-mini", { + openai: ["openai:gpt-5.4", "openai:gpt-5-mini"], + }), + ).toBe("openai:gpt-5-mini") + }) - const v1Settings = makeSettings({ - selectedModel: "model-b", + it("takes the first enabled model of any provider when the current one is gone", () => { + const settings = makeSettings({ + providers: { + anthropic: { + apiKey: "sk-test", + enabledModels: ["anthropic:claude-sonnet-5"], + grantSchemaAccess: false, + }, + }, + }) + expect( + getNextModel( + "openai:gpt-5-mini", + { anthropic: ["anthropic:claude-sonnet-5"] }, + settings, + ), + ).toBe("anthropic:claude-sonnet-5") + }) + + it("returns null when nothing is enabled", () => { + expect(getNextModel("openai:gpt-5-mini", {})).toBeNull() + }) + + it("stays on the outgoing model's provider when it still has models", () => { + // Given gpt-5.4 was just disabled while OpenAI keeps gpt-5-mini enabled + const updatedSettings = makeSettings({ + selectedModel: "openai:gpt-5.4", providers: { openai: { apiKey: "sk-test", - enabledModels: ["model-a", "model-b", "model-c"], + enabledModels: ["openai:gpt-5-mini"], + grantSchemaAccess: false, + }, + anthropic: { + apiKey: "sk-test", + enabledModels: ["anthropic:claude-opus-5"], grantSchemaAccess: false, }, }, }) - // v2: model-A and model-C removed - setModelOptions([ - { label: "B", value: "model-b", provider: "openai" }, - { label: "D", value: "model-d", provider: "openai" }, - ]) + // When picking the next model + const next = getNextModel( + "openai:gpt-5.4", + { + openai: ["openai:gpt-5-mini"], + anthropic: ["anthropic:claude-opus-5"], + }, + updatedSettings, + ) - const reconciled = reconcileSettings(v1Settings) - expect(reconciled.providers.openai!.enabledModels).toEqual(["model-b"]) - expect(reconciled.selectedModel).toBe("model-b") - expect(getSelectedModel(reconciled)).toBe("model-b") + // Then it falls back within OpenAI instead of hopping to Anthropic + expect(next).toBe("openai:gpt-5-mini") }) +}) - it("downgrade: user has models from a newer version", () => { - setModelOptions([{ label: "A", value: "model-a", provider: "openai" }]) +describe("getUtilityModel", () => { + it("returns the persisted utility model for a built-in provider", () => { + const settings = makeSettings({ + selectedModel: "openai:gpt-5.4", + providers: { + openai: { + apiKey: "sk-test", + enabledModels: ["openai:gpt-5.4"], + grantSchemaAccess: false, + utilityModel: "openai:gpt-5.6-luna", + }, + }, + }) + expect(getUtilityModel("openai", settings)).toBe("openai:gpt-5.6-luna") + }) - const futureSettings = makeSettings({ - selectedModel: "model-future", + it("falls back to the selected model when nothing is persisted", () => { + const settings = makeSettings({ + selectedModel: "openai:gpt-5.4", providers: { openai: { apiKey: "sk-test", - enabledModels: ["model-a", "model-future"], + enabledModels: ["openai:gpt-5.4"], grantSchemaAccess: false, }, }, }) + expect(getUtilityModel("openai", settings)).toBe("openai:gpt-5.4") + }) - const reconciled = reconcileSettings(futureSettings) - expect(reconciled.providers.openai!.enabledModels).toEqual(["model-a"]) - expect(reconciled.selectedModel).toBe("model-a") + it("uses the selected model for custom providers", () => { + const settings = makeSettings({ + selectedModel: "custom-1:llm-a", + customProviders: { + "custom-1": { + type: "openai-chat-completions", + name: "Test", + baseURL: "http://localhost:11434/v1", + contextWindow: 100_000, + models: ["llm-a"], + }, + }, + }) + expect(getUtilityModel("custom-1", settings)).toBe("custom-1:llm-a") }) }) @@ -394,7 +708,7 @@ describe("getAiPermissions", () => { it("returns all-false when the selected model's provider has no settings", () => { const settings = makeSettings({ - selectedModel: "gpt-5-mini", + selectedModel: "openai:gpt-5-mini", providers: {}, }) expect(getAiPermissions(settings)).toEqual({ @@ -406,11 +720,11 @@ describe("getAiPermissions", () => { it("defaults read/write to false when only the legacy grantSchemaAccess is persisted", () => { const settings = makeSettings({ - selectedModel: "gpt-5-mini", + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini"], + enabledModels: ["openai:gpt-5-mini"], grantSchemaAccess: true, }, }, @@ -424,11 +738,11 @@ describe("getAiPermissions", () => { it("returns the three booleans verbatim when all are persisted on a built-in provider", () => { const settings = makeSettings({ - selectedModel: "gpt-5-mini", + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini"], + enabledModels: ["openai:gpt-5-mini"], grantSchemaAccess: true, read: true, write: true, @@ -476,11 +790,11 @@ describe("getAiPermissions", () => { it("returns false for read when grantSchemaAccess is true but read is explicitly false", () => { const settings = makeSettings({ - selectedModel: "gpt-5-mini", + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini"], + enabledModels: ["openai:gpt-5-mini"], grantSchemaAccess: true, read: false, write: false, @@ -494,3 +808,45 @@ describe("getAiPermissions", () => { }) }) }) + +describe("buildListingMetadata", () => { + const GPT5_ERA = Date.UTC(2025, 7, 7) / 1000 + const openaiListing = [ + { id: "gpt-5.4", created: GPT5_ERA + 300 }, + { id: "gpt-5.4-nano", created: GPT5_ERA + 300 }, + { id: "gpt-5-mini", created: GPT5_ERA + 200 }, + { id: "whisper-1", created: GPT5_ERA + 100 }, + ] + + it("derives labels and a cheap utility model from an OpenAI listing", () => { + // Given enabled models including a manually added unlisted id + const metadata = buildListingMetadata("openai", openaiListing, [ + "gpt-5.4", + "my-proxy-model", + ]) + + // Then listed ids get derived labels and unlisted ids keep the raw value + expect(metadata.modelLabels).toEqual({ + "gpt-5.4": "GPT 5.4", + "my-proxy-model": "my-proxy-model", + }) + // And the utility model comes from the cheap tier of the chat pool + expect(metadata.utilityModel).toBe("openai:gpt-5.4-nano") + }) + + it("prefers provider labels and haiku-tier utility for an Anthropic listing", () => { + // Given a listing with provider display names + const listing = [ + { id: "claude-opus-5", label: "Claude Opus 5", created: 300 }, + { id: "claude-haiku-4-5", label: "Claude Haiku 4.5", created: 200 }, + ] + + const metadata = buildListingMetadata("anthropic", listing, [ + "claude-opus-5", + ]) + + // Then the stored label is the provider's and utility picks the haiku tier + expect(metadata.modelLabels).toEqual({ "claude-opus-5": "Claude Opus 5" }) + expect(metadata.utilityModel).toBe("anthropic:claude-haiku-4-5") + }) +}) diff --git a/src/utils/ai/settings.ts b/src/utils/ai/settings.ts index 65bdaf8bf..d90697097 100644 --- a/src/utils/ai/settings.ts +++ b/src/utils/ai/settings.ts @@ -1,10 +1,18 @@ import type { AiAssistantSettings, CustomProviderDefinition, + ProviderSettings, } from "../../providers/LocalStorageProvider/types" import type { Permissions } from "../tools/permissions" import { getValue } from "../localStorage" import { StoreKey } from "../localStorage/types" +import { + filterOpenAiChatModels, + formatModelLabel, + resolveUtilityModel, + UTILITY_MODEL_TIERS, +} from "./modelCatalog" +import type { ProviderModel } from "./modelCatalog" export type ProviderType = "anthropic" | "openai" | "openai-chat-completions" @@ -38,160 +46,97 @@ export type ModelOption = { label: string value: string provider: ProviderId - isSlow?: boolean - isTestModel?: boolean - default?: boolean - defaultEnabled?: boolean } -export const MODEL_OPTIONS: ModelOption[] = [ - { - label: "Claude Opus 4.7", - value: "claude-opus-4-7", - provider: "anthropic", - isSlow: true, - defaultEnabled: true, - default: true, - }, - { - label: "Claude Sonnet 4.6", - value: "claude-sonnet-4-6", - provider: "anthropic", - defaultEnabled: true, - }, - { - label: "Claude Sonnet 4.5", - value: "claude-sonnet-4-5", - provider: "anthropic", - }, - { - label: "Claude Haiku 4.5", - value: "claude-haiku-4-5", - provider: "anthropic", - isTestModel: true, - }, - { - label: "GPT-5.4 (High Reasoning)", - value: "gpt-5.4@reasoning=high", - provider: "openai", - }, - { - label: "GPT-5.4 (Medium Reasoning)", - value: "gpt-5.4@reasoning=medium", - provider: "openai", - defaultEnabled: true, - default: true, - }, - { - label: "GPT-5.4 (Low Reasoning)", - value: "gpt-5.4@reasoning=low", - provider: "openai", - defaultEnabled: true, - }, - { - label: "GPT-5 mini", - value: "gpt-5-mini", - provider: "openai", - defaultEnabled: true, - }, - { - label: "GPT-5 nano", - value: "gpt-5-nano", - provider: "openai", - defaultEnabled: true, - isTestModel: true, - }, -] - -export type ReasoningEffort = "high" | "medium" | "low" - -export type ModelProps = { - model: string - reasoningEffort?: ReasoningEffort -} - -const CUSTOM_MODEL_SEP = ":" +const MODEL_VALUE_SEP = ":" -export const makeCustomModelValue = ( +export const makeModelValue = ( providerId: ProviderId, modelId: string, -): string => `${providerId}${CUSTOM_MODEL_SEP}${modelId}` +): string => `${providerId}${MODEL_VALUE_SEP}${modelId}` export const parseModelValue = ( value: string, -): { customProviderId: string; rawModel: string } | { rawModel: string } => { - const sepIndex = value.indexOf(CUSTOM_MODEL_SEP) + customProviders?: Record, +): { providerId: ProviderId; rawModel: string } | { rawModel: string } => { + const sepIndex = value.indexOf(MODEL_VALUE_SEP) if (sepIndex === -1) return { rawModel: value } const candidateProvider = value.slice(0, sepIndex) - // Only treat as namespaced if the prefix is NOT a built-in provider. - if (BUILTIN_PROVIDERS[candidateProvider]) return { rawModel: value } + if ( + !Object.hasOwn(BUILTIN_PROVIDERS, candidateProvider) && + (!customProviders || !Object.hasOwn(customProviders, candidateProvider)) + ) { + return { rawModel: value } + } return { - customProviderId: candidateProvider, + providerId: candidateProvider, rawModel: value.slice(sepIndex + 1), } } +export const stripModelNamespace = ( + value: string, + providerId: ProviderId, +): string => { + const prefix = `${providerId}${MODEL_VALUE_SEP}` + return value.startsWith(prefix) ? value.slice(prefix.length) : value +} + +export const getModelLabel = ( + modelValue: string, + providerId: ProviderId, + settings?: AiAssistantSettings, +): string => { + const modelId = stripModelNamespace(modelValue, providerId) + return ( + settings?.providers?.[providerId]?.modelLabels?.[modelId] ?? + formatModelLabel(modelId) + ) +} + export const getAllModelOptions = ( settings?: AiAssistantSettings, ): ModelOption[] => { - if (!settings?.customProviders) return MODEL_OPTIONS - const customModels: ModelOption[] = [] - for (const [providerId, def] of Object.entries(settings.customProviders)) { + if (!settings) return [] + const options: ModelOption[] = [] + for (const providerId of Object.keys(BUILTIN_PROVIDERS)) { + const enabledModels = settings.providers?.[providerId]?.enabledModels ?? [] + for (const value of enabledModels) { + options.push({ + label: getModelLabel(value, providerId, settings), + value, + provider: providerId, + }) + } + } + for (const [providerId, def] of Object.entries( + settings.customProviders ?? {}, + )) { for (const modelId of def.models) { - customModels.push({ + options.push({ label: modelId, - value: makeCustomModelValue(providerId, modelId), + value: makeModelValue(providerId, modelId), provider: providerId, }) } } - return [...MODEL_OPTIONS, ...customModels] + return options } export const providerForModel = ( model: ModelOption["value"], - _settings?: AiAssistantSettings, + settings?: AiAssistantSettings, ): ProviderId | null => { - // Check for namespaced custom model value (providerId:modelId) - const parsed = parseModelValue(model) - if ("customProviderId" in parsed) return parsed.customProviderId - // Fall back to built-in model lookup - return MODEL_OPTIONS.find((m) => m.value === model)?.provider ?? null -} - -export const getModelProps = (model: ModelOption["value"]): ModelProps => { - const { rawModel } = parseModelValue(model) - const parts = rawModel.split("@") - const modelName = parts[0] - const extraParams = parts[1] - ?.split(",") - ?.map((p) => ({ key: p.split("=")[0], value: p.split("=")[1] })) - if (extraParams) { - const reasoningParam = extraParams.find((p) => p.key === "reasoning") - if (reasoningParam && reasoningParam.value) { - return { - model: modelName, - reasoningEffort: reasoningParam.value as ReasoningEffort, - } - } - } - return { model: modelName } + const parsed = parseModelValue(model, settings?.customProviders) + return "providerId" in parsed ? parsed.providerId : null } export const getAllProviders = ( settings?: AiAssistantSettings, -): ProviderId[] => { - const providers = new Set() - MODEL_OPTIONS.forEach((model) => { - providers.add(model.provider) - }) - if (settings?.customProviders) { - for (const id of Object.keys(settings.customProviders)) { - providers.add(id) - } - } - return Array.from(providers) -} +): ProviderId[] => [ + ...Object.keys(BUILTIN_PROVIDERS), + ...Object.keys(settings?.customProviders ?? {}), +] export const getSelectedModel = ( settings: AiAssistantSettings, @@ -205,16 +150,7 @@ export const getSelectedModel = ( ) { return selectedModel } - - const allModels = getAllModelOptions(settings) - // Fall back to first enabled default model, then first enabled model - return ( - enabledModels.find( - (id) => allModels.find((m) => m.value === id)?.default, - ) ?? - enabledModels[0] ?? - null - ) + return enabledModels[0] ?? null } export const getAllEnabledModels = ( @@ -228,7 +164,7 @@ export const getAllEnabledModels = ( } else if (settings.customProviders?.[provider]) { models.push( ...settings.customProviders[provider].models.map((m) => - makeCustomModelValue(provider, m), + makeModelValue(provider, m), ), ) } @@ -241,37 +177,27 @@ export const getNextModel = ( enabledModels: Record, settings?: AiAssistantSettings, ): string | null => { - let nextModel: string | null | undefined = currentModel - - const allModels = getAllModelOptions(settings) - const modelProvider = currentModel - ? providerForModel(currentModel, settings) - : null - if (modelProvider && enabledModels[modelProvider]?.length > 0) { - // Current model is still enabled, so we can use it - if (currentModel && enabledModels[modelProvider].includes(currentModel)) { + const providerOf = (model: string) => { + const parsed = parseModelValue(model, settings?.customProviders) + return "providerId" in parsed ? parsed.providerId : null + } + const modelProvider = currentModel ? providerOf(currentModel) : null + if ( + currentModel && + modelProvider && + enabledModels[modelProvider]?.length > 0 + ) { + if (enabledModels[modelProvider].includes(currentModel)) { return currentModel } - // Take the default model of this provider, otherwise the first enabled model of this provider - nextModel = - enabledModels[modelProvider].find( - (m) => allModels.find((mo) => mo.value === m)?.default, - ) ?? enabledModels[modelProvider][0] - } else { - // No other enabled models for this provider, we have to choose from another provider if exists - const otherProviderWithEnabledModel = getAllProviders(settings).find( - (p) => enabledModels[p]?.length > 0, - ) - if (otherProviderWithEnabledModel) { - nextModel = - enabledModels[otherProviderWithEnabledModel].find( - (m) => allModels.find((mo) => mo.value === m)?.default, - ) ?? enabledModels[otherProviderWithEnabledModel][0] - } else { - nextModel = null - } + return enabledModels[modelProvider][0] } - return nextModel ?? null + const providerWithEnabledModel = getAllProviders(settings).find( + (p) => enabledModels[p]?.length > 0, + ) + return providerWithEnabledModel + ? enabledModels[providerWithEnabledModel][0] + : null } export const isAiAssistantConfigured = ( @@ -288,19 +214,81 @@ export const canUseAiAssistant = (settings: AiAssistantSettings): boolean => { return isAiAssistantConfigured(settings) && !!settings.selectedModel } -export const getTestModel = ( +export const getUtilityModel = ( providerId: ProviderId, settings?: AiAssistantSettings, ): string | null => { - if (settings?.customProviders?.[providerId]) { + if (!settings) return null + if (settings.customProviders?.[providerId]) { return settings.selectedModel ?? null } return ( - MODEL_OPTIONS.find((m) => m.provider === providerId && m.isTestModel) - ?.value ?? null + settings.providers?.[providerId]?.utilityModel ?? getSelectedModel(settings) ) } +export type ProviderSettingsInput = { + apiKey: string + enabledModels: string[] + permissions: Permissions + modelLabels?: Record + utilityModel?: string + reasoningEffort?: "default" | "high" +} + +export const buildProviderSettings = ({ + apiKey, + enabledModels, + permissions, + modelLabels, + utilityModel, + reasoningEffort, +}: ProviderSettingsInput): ProviderSettings => ({ + apiKey, + enabledModels, + grantSchemaAccess: permissions.grantSchemaAccess, + read: permissions.read, + write: permissions.write, + ...(modelLabels && Object.keys(modelLabels).length > 0 + ? { modelLabels } + : {}), + ...(utilityModel ? { utilityModel } : {}), + ...(reasoningEffort === "high" ? { reasoningEffort: "high" as const } : {}), +}) + +/** + * Derives the listing-dependent provider settings captured at Save time: + * labels for enabled models, the utility model, and the reasoning gate. + */ +export type ListingMetadata = { + modelLabels: Record + utilityModel?: string +} + +export const buildListingMetadata = ( + providerId: ProviderId, + listing: ProviderModel[], + enabledModels: string[], +): ListingMetadata => { + const isOpenAi = BUILTIN_PROVIDERS[providerId]?.type === "openai" + const utilityPool = isOpenAi ? filterOpenAiChatModels(listing) : listing + const utilityModel = resolveUtilityModel( + utilityPool, + UTILITY_MODEL_TIERS[isOpenAi ? "openai" : "anthropic"], + ) + const modelLabels: Record = {} + for (const id of enabledModels) { + const listed = listing.find((m) => m.id === id) + modelLabels[id] = listed ? (listed.label ?? formatModelLabel(id)) : id + } + return { + modelLabels, + ...(utilityModel + ? { utilityModel: makeModelValue(providerId, utilityModel) } + : {}), + } +} + /** * Returns the context window for a given provider. * For custom providers, returns the configured value. @@ -314,41 +302,6 @@ export const getProviderContextWindow = ( return custom?.contextWindow ?? null } -/** - * Reconciles persisted AI assistant settings against current model options. - * Removes stale model IDs from built-in providers' enabledModels. - * Preserves custom provider models (validated against customProviders definitions). - * - * Pure function — does not write to localStorage. - * Idempotent: applying it multiple times produces the same result. - */ -export const reconcileSettings = ( - settings: AiAssistantSettings, -): AiAssistantSettings => { - const allValidIds = new Set(getAllModelOptions(settings).map((m) => m.value)) - const result = { - ...settings, - providers: { ...settings.providers }, - } - - for (const providerKey of Object.keys(result.providers)) { - const providerSettings = result.providers[providerKey] - if (!providerSettings?.enabledModels) continue - - const models = providerSettings.enabledModels.filter((id) => - allValidIds.has(id), - ) - result.providers[providerKey] = { - ...providerSettings, - enabledModels: models, - } - } - - result.selectedModel = getSelectedModel(result) ?? undefined - - return result -} - export const getApiKey = ( providerId: ProviderId, settings: AiAssistantSettings, @@ -395,17 +348,18 @@ export const getAiPermissions = ( } export const readLiveAiAssistantSettings = (): AiAssistantSettings | null => { - const stored = getValue(StoreKey.AI_ASSISTANT_SETTINGS) - if (!stored) return null try { + const stored = getValue(StoreKey.AI_ASSISTANT_SETTINGS) + if (!stored) return null const parsed = JSON.parse(stored) as AiAssistantSettings - return reconcileSettings({ + return { + modelValueFormat: parsed.modelValueFormat, selectedModel: parsed.selectedModel, providers: parsed.providers || {}, ...(parsed.customProviders && { customProviders: parsed.customProviders, }), - }) + } } catch { return null } diff --git a/src/utils/ai/types.ts b/src/utils/ai/types.ts index e0950b921..4f1a7e5f8 100644 --- a/src/utils/ai/types.ts +++ b/src/utils/ai/types.ts @@ -8,8 +8,11 @@ import type { import type { Permissions, ToolCategory } from "../tools/permissions" import type { ValidateQueryResult } from "../questdb/types" import type { ProviderId } from "./settings" +import type { ProviderModel } from "./modelCatalog" import type { ToolExecutionContext } from "./shared" +export { type ProviderModel } + export type ToolSurface = "ai" | "mcp" export interface ToolDefinition { @@ -97,18 +100,13 @@ export interface AIProvider { abortSignal?: AbortSignal }): Promise - testConnection(params: { - apiKey: string - model: string - }): Promise<{ valid: boolean; error?: string }> - countTokens(params: { messages: Message[] systemPrompt: string model: string }): Promise - listModels(): Promise + listModels(): Promise classifyError(error: unknown, setStatus: StatusCallback): AiAssistantAPIError isNonRetryableError(error: unknown): boolean diff --git a/src/utils/localStorage/migrate.ts b/src/utils/localStorage/migrate.ts new file mode 100644 index 000000000..4f1d00009 --- /dev/null +++ b/src/utils/localStorage/migrate.ts @@ -0,0 +1,163 @@ +import { + AI_MODEL_VALUE_FORMAT, + type AiAssistantSettings, + type CustomProviderDefinition, +} from "../../providers/LocalStorageProvider/types" +import { StoreKey } from "./types" + +type StorageAccess = Pick + +const BUILTIN_PROVIDER_IDS = ["anthropic", "openai"] as const +const LEGACY_REASONING_VARIANT = /@reasoning=(high|medium|low)$/ + +const makeModelValue = (providerId: string, modelId: string): string => + `${providerId}:${modelId}` + +const stripModelNamespace = (value: string, providerId: string): string => { + const prefix = `${providerId}:` + return value.startsWith(prefix) ? value.slice(prefix.length) : value +} + +const collapseLegacyVariant = (modelId: string): string => + modelId.replace(LEGACY_REASONING_VARIANT, "") + +const isBuiltinProvider = (providerId: string): boolean => + BUILTIN_PROVIDER_IDS.some((candidate) => candidate === providerId) + +const selectedProviderFromLegacySettings = ( + settings: AiAssistantSettings, +): { providerId: string; modelId: string } | null => { + const selectedModel = settings.selectedModel + if (!selectedModel) return null + + const separatorIndex = selectedModel.indexOf(":") + const prefix = + separatorIndex === -1 ? null : selectedModel.slice(0, separatorIndex) + if (prefix && Object.hasOwn(settings.customProviders ?? {}, prefix)) { + return { + providerId: prefix, + modelId: selectedModel.slice(separatorIndex + 1), + } + } + + const providerId = BUILTIN_PROVIDER_IDS.find((candidate) => + settings.providers[candidate]?.enabledModels?.includes(selectedModel), + ) + return providerId ? { providerId, modelId: selectedModel } : null +} + +const getEnabledModels = (settings: AiAssistantSettings): string[] => { + const providerIds = [ + ...BUILTIN_PROVIDER_IDS, + ...Object.keys(settings.customProviders ?? {}), + ] + return providerIds.flatMap((providerId) => { + const enabledModels = settings.providers[providerId]?.enabledModels + if (enabledModels) return enabledModels + return (settings.customProviders?.[providerId]?.models ?? []).map( + (modelId) => makeModelValue(providerId, modelId), + ) + }) +} + +/** Migrates the unversioned local-storage schema to the current schema. */ +export const migrateLocalStorage = ( + storage: StorageAccess = localStorage, +): boolean => { + try { + const stored = storage.getItem(StoreKey.AI_ASSISTANT_SETTINGS) + if (!stored) return true + + const settings = JSON.parse(stored) as AiAssistantSettings + + // Only the old, unversioned format is ours to migrate. Never reinterpret a + // current or future version. + if (settings.modelValueFormat !== undefined) return true + + const selected = selectedProviderFromLegacySettings(settings) + const migratedProviders: AiAssistantSettings["providers"] = { + ...settings.providers, + } + + for (const [providerId, providerSettings] of Object.entries( + settings.providers, + )) { + if (!providerSettings?.enabledModels) continue + + const builtin = isBuiltinProvider(providerId) + const customProvider: CustomProviderDefinition | undefined = + settings.customProviders?.[providerId] + const validCustomModels = customProvider + ? new Set(customProvider.models) + : null + const rawEnabledModels = providerSettings.enabledModels.map((value) => + builtin ? value : stripModelNamespace(value, providerId), + ) + const selectedHighVariant = + builtin && + selected?.providerId === providerId && + selected.modelId.endsWith("@reasoning=high") && + rawEnabledModels.includes(selected.modelId) + const enabledModels = [ + ...new Set( + rawEnabledModels + .map((modelId) => + builtin ? collapseLegacyVariant(modelId) : modelId, + ) + .filter((modelId) => builtin || validCustomModels?.has(modelId)) + .map((modelId) => makeModelValue(providerId, modelId)), + ), + ] + const rawUtilityModel = providerSettings.utilityModel + ? builtin + ? providerSettings.utilityModel + : stripModelNamespace(providerSettings.utilityModel, providerId) + : null + + migratedProviders[providerId] = { + ...providerSettings, + enabledModels, + ...(rawUtilityModel + ? { + utilityModel: makeModelValue( + providerId, + builtin + ? collapseLegacyVariant(rawUtilityModel) + : rawUtilityModel, + ), + } + : {}), + ...(selectedHighVariant ? { reasoningEffort: "high" as const } : {}), + } + } + + const migrated: AiAssistantSettings = { + ...settings, + modelValueFormat: AI_MODEL_VALUE_FORMAT, + providers: migratedProviders, + ...(selected + ? { + selectedModel: makeModelValue( + selected.providerId, + isBuiltinProvider(selected.providerId) + ? collapseLegacyVariant(selected.modelId) + : selected.modelId, + ), + } + : { selectedModel: undefined }), + } + const enabledModels = getEnabledModels(migrated) + if ( + !migrated.selectedModel || + !enabledModels.includes(migrated.selectedModel) + ) { + migrated.selectedModel = enabledModels[0] + } + + storage.setItem(StoreKey.AI_ASSISTANT_SETTINGS, JSON.stringify(migrated)) + return true + } catch { + // Leave invalid persisted data untouched; the regular reader uses defaults. + return false + } +}