From 1665906d7cc47433a95039e76b721fdb6c2127b3 Mon Sep 17 00:00:00 2001 From: emrberk Date: Thu, 3 Sep 2026 19:32:09 +0300 Subject: [PATCH 1/9] feat: dynamic model listing, single enablement step, provider-level reasoning control Replace the fixed built-in model list with the providers' live /v1/models listings. Availability now comes from the listing; pure heuristics derive labels, filter OpenAI noise (blocklist-only, with a Show-all escape hatch), resolve a utility model for chat titles (haiku->sonnet / luna->nano->mini, newest first), and gate reasoning-capable OpenAI models by creation date. - Manage Models becomes the single enablement step for every provider; the settings modal body is a read-only summary and flags models the provider removed ("No longer available", dropped on Save) - API key validation is a listing fetch; the chat-ping validation chain and the test-model flag are gone - Reasoning simplifies to one provider-level Default/High dropdown for OpenAI (applied to chat and compaction, never titles, with a stateless 400-strip-retry); Anthropic needs no control since modern Claude runs adaptive thinking by default. Legacy @reasoning= variants collapse to plain ids on load and a selected high variant folds into the new setting - Context compaction runs on the selected model; titles use the utility model - Drop temperature from Anthropic requests: every 4.7+/5.x model now rejects it as deprecated (verified against all 11 listed models) - Remove the slow-model/brain-icon story Verified live against both provider APIs: request shapes pass on all 11 Anthropic and all 30 listed OpenAI chat models (Default and High). Co-Authored-By: Claude Fable 5 --- e2e/commands.js | 3 +- e2e/questdb | 2 +- e2e/tests/console/aiAssistant.spec.js | 210 ++++++----- src/components/AIStatusIndicator/index.tsx | 56 +-- .../SetupAIAssistant/ConfigurationModal.tsx | 266 +++++++------- .../SetupAIAssistant/ManageModelsModal.tsx | 290 +++++++++++++-- .../SetupAIAssistant/ModelDropdown.tsx | 3 - .../SetupAIAssistant/ModelPicker.tsx | 332 ++++++++++++++++++ .../SetupAIAssistant/ModelSettings.tsx | 251 ++++--------- .../SetupAIAssistant/ReasoningSection.tsx | 87 +++++ .../SetupAIAssistant/SettingsModal.tsx | 268 ++++++++------ src/providers/LocalStorageProvider/types.ts | 4 + .../MCPBridgeStatus/PermissionsSection.tsx | 10 +- src/utils/ai/aiAssistant.ts | 10 - src/utils/ai/anthropicProvider.ts | 70 +--- src/utils/ai/contextCompaction.ts | 10 +- src/utils/ai/executeAIFlow.ts | 8 +- src/utils/ai/index.ts | 16 +- src/utils/ai/modelCatalog.test.ts | 167 +++++++++ src/utils/ai/modelCatalog.ts | 119 +++++++ src/utils/ai/openaiChatCompletionsProvider.ts | 153 ++++---- src/utils/ai/openaiProvider.ts | 185 +++++----- src/utils/ai/openaiShared.ts | 9 + src/utils/ai/registry.ts | 24 +- src/utils/ai/settings.test.ts | 305 ++++++++-------- src/utils/ai/settings.ts | 288 +++++++-------- src/utils/ai/types.ts | 10 +- 27 files changed, 1975 insertions(+), 1181 deletions(-) create mode 100644 src/components/SetupAIAssistant/ModelPicker.tsx create mode 100644 src/components/SetupAIAssistant/ReasoningSection.tsx create mode 100644 src/utils/ai/modelCatalog.test.ts create mode 100644 src/utils/ai/modelCatalog.ts diff --git a/e2e/commands.js b/e2e/commands.js index b02401e76..06ed42418 100644 --- a/e2e/commands.js +++ b/e2e/commands.js @@ -4,8 +4,7 @@ require("@4tw/cypress-drag-drop") const { ctrlOrCmd, escapeRegExp, seedNotebookOnboarding } = require("./utils") -const contextPath = process.env.QDB_HTTP_CONTEXT_WEB_CONSOLE || "" -const baseUrl = `http://localhost:9999${contextPath}` +const baseUrl = Cypress.config("baseUrl") const tableSchemas = { btc_trades: diff --git a/e2e/questdb b/e2e/questdb index 9b59a9211..090a22720 160000 --- a/e2e/questdb +++ b/e2e/questdb @@ -1 +1 @@ -Subproject commit 9b59a921165af573cedd22bf8b12613de19cb8bd +Subproject commit 090a2272082337f5f986b80e348f9e9d659ab08f diff --git a/e2e/tests/console/aiAssistant.spec.js b/e2e/tests/console/aiAssistant.spec.js index e362d6e54..06d98d7aa 100644 --- a/e2e/tests/console/aiAssistant.spec.js +++ b/e2e/tests/console/aiAssistant.spec.js @@ -81,29 +81,35 @@ function interceptAIChatRequest( } /** - * Intercepts AI provider token validation requests. + * Intercepts AI provider model listing requests. + * Validation now runs through GET /v1/models, so a mocked listing both + * validates the key and feeds the model picker. + * + * The OpenAI listing carries noise (whisper-1) that the picker must filter, + * and `created` timestamps that drive newest-first ordering. * * @param {"anthropic" | "openai"} provider - The AI provider to intercept - * @param {boolean} success - If true, returns 200 success response; if false, returns 401 error + * @param {boolean} success - If true, returns 200 with a listing; if false, returns 401 */ function interceptTokenValidation(provider, success) { - const endpoint = PROVIDERS[provider].endpoint - if (provider === "openai") { if (success) { - cy.intercept("POST", endpoint, { + cy.intercept("GET", "https://api.openai.com/v1/models*", { statusCode: 200, delay: 200, body: { - id: "resp_mock_test", - object: "response", - created_at: Date.now(), - status: "completed", - output: [], + object: "list", + data: [ + { id: "gpt-5.4", object: "model", created: 1772000000 }, + { id: "gpt-5-mini", object: "model", created: 1754500000 }, + { id: "gpt-5", object: "model", created: 1754400000 }, + { id: "gpt-5-nano", object: "model", created: 1754300000 }, + { id: "whisper-1", object: "model", created: 1677532384 }, + ], }, }).as("openaiValidation") } else { - cy.intercept("POST", endpoint, { + cy.intercept("GET", "https://api.openai.com/v1/models*", { statusCode: 401, delay: 200, body: { @@ -119,24 +125,37 @@ function interceptTokenValidation(provider, success) { } } else if (provider === "anthropic") { if (success) { - cy.intercept("POST", endpoint, { + cy.intercept("GET", "https://api.anthropic.com/v1/models*", { statusCode: 200, delay: 200, body: { - id: "msg_mock_test", - type: "message", - role: "assistant", - content: [], - model: "claude-sonnet-4-5", - stop_reason: "end_turn", - usage: { - input_tokens: 10, - output_tokens: 5, - }, + data: [ + { + type: "model", + id: "claude-opus-4-5", + display_name: "Claude Opus 4.5", + created_at: "2025-11-01T00:00:00Z", + }, + { + type: "model", + id: "claude-sonnet-4-5", + display_name: "Claude Sonnet 4.5", + created_at: "2025-09-29T00:00:00Z", + }, + { + type: "model", + id: "claude-haiku-4-5", + display_name: "Claude Haiku 4.5", + created_at: "2025-10-01T00:00:00Z", + }, + ], + has_more: false, + first_id: "claude-opus-4-5", + last_id: "claude-haiku-4-5", }, }).as("anthropicValidation") } else { - cy.intercept("POST", endpoint, { + cy.intercept("GET", "https://api.anthropic.com/v1/models*", { statusCode: 401, delay: 200, body: { @@ -165,6 +184,14 @@ describe("ai assistant", () => { `Unhandled Anthropic request detected! Request body: ${JSON.stringify(req.body).slice(0, 200)}...`, ) }).as("unhandledAnthropic") + + cy.intercept("GET", "https://api.openai.com/v1/models*", () => { + throw new Error("Unhandled OpenAI model listing request detected!") + }).as("unhandledOpenAIModels") + + cy.intercept("GET", "https://api.anthropic.com/v1/models*", () => { + throw new Error("Unhandled Anthropic model listing request detected!") + }).as("unhandledAnthropicModels") }) describe("onboarding and settings", () => { @@ -294,10 +321,15 @@ describe("ai assistant", () => { cy.getByDataHook("ai-settings-api-key").type("valid-api-key") cy.getByDataHook("multi-step-modal-next-button").click() - // Then + // Then - step two shows the filtered listing, nothing preselected cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") + cy.getByDataHook("configure-models-model-row").should("have.length", 4) - // When + // When - enable two models and activate + cy.getByDataHook("configure-models-model-row").contains("gpt-5.4").click() + cy.getByDataHook("configure-models-model-row") + .contains("gpt-5-mini") + .click() cy.getByDataHook("multi-step-modal-next-button").click() // Then @@ -396,7 +428,10 @@ describe("ai assistant", () => { // Then cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") - // When - drop permissions to None so schema tools are excluded. + // When - enable a model, drop permissions to None so schema tools are excluded. + cy.getByDataHook("configure-models-model-row") + .contains("gpt-5-mini") + .click() cy.getByDataHook("permissions-trigger").click() cy.getByDataHook("permission-level-none").click() cy.getByDataHook("multi-step-modal-next-button").click() @@ -444,8 +479,8 @@ describe("ai assistant", () => { }) it("should work with multiple providers", () => { - const openaiEnabledModels = [] - const anthropicEnabledModels = [] + const openaiEnabledModels = ["GPT-5.4", "GPT-5 Mini"] + const anthropicEnabledModels = ["Claude Opus 4.5", "Claude Sonnet 4.5"] // Given - Set up OpenAI provider first interceptTokenValidation("openai", true) @@ -462,10 +497,11 @@ describe("ai assistant", () => { // Then - Should be on step two cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") - // When - Store enabled model labels for OpenAI - cy.get('[data-model-enabled="true"]').each(($modelRow) => { - openaiEnabledModels.push($modelRow.attr("data-model")) - }) + // When - Enable two OpenAI models + cy.getByDataHook("configure-models-model-row").contains("gpt-5.4").click() + cy.getByDataHook("configure-models-model-row") + .contains("gpt-5-mini") + .click() cy.getByDataHook("multi-step-modal-next-button").click() @@ -500,19 +536,24 @@ describe("ai assistant", () => { cy.getByDataHook("ai-settings-api-key").type("valid-anthropic-key") cy.getByDataHook("ai-settings-test-api").click() - // Then - Should show validating and then validated + // Then - Validation opens Manage Models with the fetched listing cy.wait("@anthropicValidation") + cy.getByDataHook("manage-models-model-row").should("have.length", 3) + + // When - Enable two Anthropic models and save the picker + cy.getByDataHook("manage-models-model-row") + .contains("Claude Opus 4.5") + .click() + cy.getByDataHook("manage-models-model-row") + .contains("Claude Sonnet 4.5") + .click() + cy.getByDataHook("manage-models-save").click() // Then - Anthropic should no longer show Inactive cy.getByDataHook("ai-settings-provider-anthropic") .getByDataHook("ai-settings-provider-status") .should("not.contain", "Inactive") - // When - Store enabled model labels for Anthropic - cy.get('[data-enabled="true"]').each(($modelRow) => { - anthropicEnabledModels.push($modelRow.attr("data-model")) - }) - // When - Save settings cy.getByDataHook("ai-settings-save").click() cy.get(".toast-success-container").should("be.visible").click() @@ -3499,15 +3540,32 @@ describe("custom providers", () => { cy.contains("codellama").should("be.visible") cy.get("body").type("{esc}") // close dropdown + cy.intercept("GET", "http://localhost:11434/v1/models*", { + statusCode: 200, + body: { + object: "list", + data: [ + { id: "llama3", object: "model" }, + { id: "mistral", object: "model" }, + { id: "codellama", object: "model" }, + ], + }, + }).as("ollamaModels") + cy.getByDataHook("ai-assistant-settings-button").click() cy.getByDataHook("ai-settings-provider-ollama").should("be.visible").click() + // Models render as read-only rows; changes go through Manage models cy.get("[data-model='llama3']").should("exist") cy.get("[data-model='mistral']").should("exist") cy.get("[data-model='codellama']").should("exist") - cy.get("[data-model='mistral']").find("button[role='switch']").click() - cy.get("[data-model='mistral'][data-enabled='false']").should("exist") + cy.getByDataHook("ai-settings-manage-models").click() + cy.wait("@ollamaModels") + cy.getByDataHook("custom-provider-model-row").contains("mistral").click() + cy.getByDataHook("manage-models-save").click() + + cy.get("[data-model='mistral']").should("not.exist") cy.getByDataHook("ai-settings-save").click() cy.getByDataHook("ai-settings-model-dropdown").should("be.visible").click() @@ -3518,10 +3576,14 @@ describe("custom providers", () => { cy.getByDataHook("ai-assistant-settings-button").click() cy.getByDataHook("ai-settings-provider-ollama").click() - cy.get("[data-model='mistral'][data-enabled='false']").should("exist") + cy.get("[data-model='mistral']").should("not.exist") - cy.get("[data-model='mistral']").find("button[role='switch']").click() - cy.get("[data-model='mistral'][data-enabled='true']").should("exist") + cy.getByDataHook("ai-settings-manage-models").click() + cy.wait("@ollamaModels") + cy.getByDataHook("custom-provider-model-row").contains("mistral").click() + cy.getByDataHook("manage-models-save").click() + + cy.get("[data-model='mistral']").should("exist") cy.getByDataHook("ai-settings-save").click() cy.getByDataHook("ai-settings-model-dropdown").should("be.visible").click() @@ -3859,7 +3921,7 @@ describe("custom providers", () => { cy.getByDataHook("ai-settings-model-dropdown").click() cy.getByDataHook("ai-settings-model-item-label") - .contains("GPT-5 mini") + .contains("GPT-5 Mini") .click() cy.getByDataHook("chat-window-new").click() @@ -4158,7 +4220,7 @@ describe("custom providers", () => { cy.get("body").type("{esc}") // close dropdown }) - it("should auto-enable new models from manage models and preserve unsaved toggle state", () => { + it("should add and remove models through manage models in manual mode", () => { const providerId = "test-provider" cy.loadConsoleWithAuth( @@ -4175,14 +4237,10 @@ describe("custom providers", () => { .should("be.visible") .click() - // All 3 models should be enabled - cy.get("[data-model='model-a'][data-enabled='true']").should("exist") - cy.get("[data-model='model-b'][data-enabled='true']").should("exist") - cy.get("[data-model='model-c'][data-enabled='true']").should("exist") - - // Disable model-b toggle (unsaved state) - cy.get("[data-model='model-b']").find("button[role='switch']").click() - cy.get("[data-model='model-b'][data-enabled='false']").should("exist") + // All 3 models render as read-only rows + cy.get("[data-model='model-a']").should("exist") + cy.get("[data-model='model-b']").should("exist") + cy.get("[data-model='model-c']").should("exist") // Intercept model fetch → fail to get manual mode cy.intercept("GET", "**/models*", { @@ -4214,10 +4272,10 @@ describe("custom providers", () => { cy.getByDataHook("manage-models-save").click() // Back in SettingsModal: model-b gone, model-d auto-enabled - cy.get("[data-model='model-a'][data-enabled='true']").should("exist") + cy.get("[data-model='model-a']").should("exist") cy.get("[data-model='model-b']").should("not.exist") - cy.get("[data-model='model-c'][data-enabled='true']").should("exist") - cy.get("[data-model='model-d'][data-enabled='true']").should("exist") + cy.get("[data-model='model-c']").should("exist") + cy.get("[data-model='model-d']").should("exist") // Save settings cy.getByDataHook("ai-settings-save").click() @@ -4234,10 +4292,6 @@ describe("custom providers", () => { it("should handle no-API-key custom provider: models visible, no validated badge, schema toggle enabled, and allow adding an API key", () => { const providerId = "ollama" - const customEndpoint = getCustomProviderEndpoint( - CUSTOM_PROVIDER_DEFAULTS.baseURL, - "openai-chat-completions", - ) cy.loadConsoleWithAuth( false, @@ -4265,7 +4319,7 @@ describe("custom providers", () => { "This provider does not have an API key", ) - // Model list visible with both models + // Model list visible with both models as read-only rows cy.get("[data-model='llama3']").should("exist") cy.get("[data-model='mistral']").should("exist") @@ -4275,11 +4329,7 @@ describe("custom providers", () => { // Manage models button visible cy.getByDataHook("ai-settings-manage-models").should("be.visible") - // Toggle mistral off - cy.get("[data-model='mistral']").find("button[role='switch']").click() - cy.get("[data-model='mistral'][data-enabled='false']").should("exist") - - // Built-in provider should NOT have manage models button + // Built-in provider should NOT have manage models button before validation cy.getByDataHook("ai-settings-provider-openai").click() cy.getByDataHook("ai-settings-manage-models").should("not.exist") @@ -4292,21 +4342,16 @@ describe("custom providers", () => { cy.getByDataHook("ai-settings-edit-api-key").click() cy.getByDataHook("ai-settings-api-key").type("sk-custom-key-123") - // Intercept validation request to custom endpoint - cy.intercept("POST", customEndpoint, { + // Validation runs through the provider's model listing + cy.intercept("GET", "http://localhost:11434/v1/models*", { statusCode: 200, delay: 200, body: { - id: "chatcmpl-mock", - object: "chat.completion", - choices: [ - { - index: 0, - message: { role: "assistant", content: "" }, - finish_reason: "stop", - }, + object: "list", + data: [ + { id: "llama3", object: "model" }, + { id: "mistral", object: "model" }, ], - usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, }, }).as("customValidation") @@ -4317,19 +4362,20 @@ describe("custom providers", () => { // Validated badge should now appear cy.getByDataHook("ai-settings-validated-badge").should("be.visible") - // Models still visible, mistral toggle preserved - cy.get("[data-model='llama3'][data-enabled='true']").should("exist") - cy.get("[data-model='mistral'][data-enabled='false']").should("exist") + // Models still visible + cy.get("[data-model='llama3']").should("exist") + cy.get("[data-model='mistral']").should("exist") // Part C: Save and verify cy.getByDataHook("ai-settings-save").click() cy.get(".toast-success-container").should("be.visible").click() - // Dropdown should show only llama3 (mistral was disabled) + // Dropdown should show both models cy.getByDataHook("ai-settings-model-dropdown").should("be.visible").click() - cy.getByDataHook("ai-settings-model-item").should("have.length", 1) + cy.getByDataHook("ai-settings-model-item").should("have.length", 2) cy.contains("llama3").should("be.visible") + cy.contains("mistral").should("be.visible") cy.get("body").type("{esc}") // close dropdown }) }) diff --git a/src/components/AIStatusIndicator/index.tsx b/src/components/AIStatusIndicator/index.tsx index ec0cdf68c..66a2fa4d7 100644 --- a/src/components/AIStatusIndicator/index.tsx +++ b/src/components/AIStatusIndicator/index.tsx @@ -11,11 +11,9 @@ import { color } from "../../utils" import { slideAnimation } from "../Animation" import { AISparkle } from "../AISparkle" import { brandLinearGradientHorizontal } from "../../theme" -import { getAllModelOptions } from "../../utils/ai" import { useAIConversation } from "../../providers/AIConversationProvider" import { Button } from "../../components/Button" import { AIStopButton } from "../AIStopButton" -import { BrainIcon } from "../SetupAIAssistant/BrainIcon" import { AssistantModes, buildOperationSections } from "./AssistantModes" import { CircleNotchSpinner } from "../../scenes/Editor/Monaco/icons" import { useSelector } from "react-redux" @@ -220,34 +218,6 @@ const ChevronButton = styled(Button).attrs({ variant: "ghost" })` margin-right: 1rem; ` -const ExtendedThinkingLabel = styled.div` - display: flex; - gap: 0.8rem; - align-items: center; - justify-content: center; - width: 100%; - flex-shrink: 0; -` - -const BrainIconWrapper = styled.div` - width: 1.6rem; - height: 1.6rem; - flex-shrink: 0; - display: flex; - align-items: center; - justify-content: center; -` - -const ExtendedThinkingText = styled.p` - flex: 1 0 0; - font-weight: 400; - font-size: 1.1rem; - color: ${color("contentSecondary")}; - min-height: 0; - min-width: 0; - margin: 0; -` - const AssistantModesContainer = styled.div` display: flex; flex-direction: column; @@ -264,14 +234,8 @@ const AssistantModesContainer = styled.div` ` export const AIStatusIndicator: React.FC = () => { - const { - status, - currentOperation, - currentModel, - abortOperation, - clearOperation, - aiAssistantSettings, - } = useAIStatus() + const { status, currentOperation, abortOperation, clearOperation } = + useAIStatus() const { chatWindowState, openChatWindow } = useAIConversation() const [expanded, setExpanded] = useState(false) const [isClosed, setIsClosed] = useState(false) @@ -280,11 +244,6 @@ export const AIStatusIndicator: React.FC = () => { const assistantModesRef = useRef(null) const activeSidebar = useSelector(selectors.console.getActiveSidebar) const statusRef = useRef(null) - const hasExtendedThinking = useMemo(() => { - return getAllModelOptions(aiAssistantSettings).find( - (model) => model.value === currentModel, - )?.isSlow - }, [currentModel, aiAssistantSettings]) const operationSections = useMemo( () => buildOperationSections(currentOperation, status, true), @@ -416,17 +375,6 @@ export const AIStatusIndicator: React.FC = () => { )} - {hasExtendedThinking && ( - - - - - - Extended thinking model enabled. Responses may be slow. - - - )} - {expanded && ( theme.color.contentPrimary}; ` -const ModelToggleRow = styled(Box).attrs({ - justifyContent: "space-between", - align: "center", - gap: "2.4rem", -})` - width: 100%; -` - -const ModelInfoColumn = styled(Box).attrs({ - flexDirection: "column", - gap: "0.8rem", -})` - flex: 1; - align-items: flex-start; -` - -const ModelInfoRow = styled(Box).attrs({ - gap: "0.8rem", - align: "center", -})` - width: 100%; -` - -const ModelDescriptionText = styled(Text)` - font-size: 1.1rem; - color: ${({ theme }) => theme.color.contentSecondary}; - flex: 1; -` - -const ModelNameText = styled(Text)` - font-size: 1.4rem; - font-weight: 400; - color: ${({ theme }) => theme.color.contentPrimary}; -` - const WarningText = styled(Text)` font-size: 1.3rem; font-weight: 400; @@ -280,10 +245,14 @@ type StepOneContentProps = { type StepTwoContentProps = { selectedProvider: ProviderId | null + listing: ProviderModel[] | null enabledModels: string[] + manualInput: string + reasoningEffortLevel: ReasoningEffortLevel permissions: Permissions - modelsByProvider: Record - onModelToggle: (modelValue: string) => void + onSelectionChange: (models: string[]) => void + onManualInputChange: (value: string) => void + onReasoningEffortChange: (next: ReasoningEffortLevel) => void onPermissionsChange: (next: Permissions) => void } @@ -417,10 +386,14 @@ const StepOneContent = ({ const StepTwoContent = ({ selectedProvider, + listing, enabledModels, + manualInput, + reasoningEffortLevel, permissions, - modelsByProvider, - onModelToggle, + onSelectionChange, + onManualInputChange, + onReasoningEffortChange, onPermissionsChange, }: StepTwoContentProps) => { const theme = useTheme() @@ -428,9 +401,18 @@ const StepTwoContent = ({ const handleClose: () => void = navigation.handleClose const currentProvider = selectedProvider - const getModelsForProvider = (provider: ProviderId) => { - return modelsByProvider[provider] || [] - } + const isOpenAi = currentProvider === "openai" + const pickerModels = listing + ? isOpenAi + ? filterOpenAiChatModels(listing) + : sortModelsNewestFirst(listing) + : [] + const hiddenModels = + listing && isOpenAi + ? sortModelsNewestFirst( + listing.filter((m) => !pickerModels.some((p) => p.id === m.id)), + ) + : undefined return ( @@ -439,9 +421,9 @@ const StepTwoContent = ({ Setup your model preferences - Enable and disable each of the models QuestDB currently supports - from this provider, and a level of data access. You'll be - able to update these settings any time. + Enable the models you want to use from this provider, and a level + of data access. You'll be able to update these settings any + time. @@ -449,11 +431,11 @@ const StepTwoContent = ({ - {currentProvider ? ( + {currentProvider && listing ? ( - Enable Models + Models {currentProvider === "anthropic" ? ( - - {getModelsForProvider(currentProvider).map((model) => { - const isEnabled = enabledModels.includes(model.value) - return ( - - - {model.label} - {model.isSlow && ( - - - - Due to advanced reasoning & thinking - capabilities, responses using this model can be - slow. - - - )} - - onModelToggle(model.value)} - data-checked={isEnabled} - /> - - ) - })} - + model.label ?? formatModelLabel(model.id)} + onSelectionChange={onSelectionChange} + onManualInputChange={onManualInputChange} + /> ) : ( @@ -515,6 +476,17 @@ const StepTwoContent = ({ )} + {isOpenAi && ( + <> + + + + + + )} {currentProvider && ( @@ -557,20 +529,15 @@ export const ConfigurationModal = ({ }, [open]) const [enabledModels, setEnabledModels] = useState([]) + const [manualInput, setManualInput] = useState("") + const [providerListing, setProviderListing] = useState< + ProviderModel[] | null + >(null) + const [reasoningEffortLevel, setReasoningEffortLevel] = + useState("default") const [permissions, setPermissions] = useState(DEFAULT_PERMISSIONS) - const modelsByProvider = useMemo(() => { - const result: Record = {} - getAllModelOptions(aiAssistantSettings).forEach((model) => { - if (!result[model.provider]) { - result[model.provider] = [] - } - result[model.provider].push(model) - }) - return result - }, [aiAssistantSettings]) - const handleProviderSelect = useCallback((provider: ProviderId) => { setSelectedProvider(provider) setError(null) @@ -582,21 +549,20 @@ export const ConfigurationModal = ({ setError(null) }, []) - const handleModelToggle = useCallback((modelValue: string) => { - setEnabledModels((prev) => { - const isEnabled = prev.includes(modelValue) - return isEnabled - ? prev.filter((m) => m !== modelValue) - : [...prev, modelValue] - }) - }, []) - const handlePermissionsChange = useCallback((next: Permissions) => { setPermissions(next) }, []) + const effectiveEnabledModels = useCallback(() => { + const pending = manualInput.trim() + return pending && !enabledModels.includes(pending) + ? [...enabledModels, pending] + : enabledModels + }, [enabledModels, manualInput]) + const handleComplete = () => { - if (!selectedProvider || enabledModels.length === 0) return + const models = effectiveEnabledModels() + if (!selectedProvider || models.length === 0) return void trackEvent(ConsoleEvent.AI_PROVIDER_CONFIGURE, { name: selectedProvider, @@ -605,24 +571,33 @@ export const ConfigurationModal = ({ write: permissions.write, }) - const selectedModel = - enabledModels.find( - (m) => - getAllModelOptions(aiAssistantSettings).find((mo) => mo.value === m) - ?.default, - ) ?? enabledModels[0] + const metadata = providerListing + ? buildListingMetadata(selectedProvider, providerListing, models) + : null const newSettings = { ...aiAssistantSettings, - selectedModel, + selectedModel: models[0], providers: { ...aiAssistantSettings.providers, [selectedProvider]: { apiKey, - enabledModels, + enabledModels: models, grantSchemaAccess: permissions.grantSchemaAccess, read: permissions.read, write: permissions.write, + ...(metadata && Object.keys(metadata.modelLabels).length > 0 + ? { modelLabels: metadata.modelLabels } + : {}), + ...(metadata?.utilityModel + ? { utilityModel: metadata.utilityModel } + : {}), + ...(metadata?.reasoningModels?.length + ? { reasoningModels: metadata.reasoningModels } + : {}), + ...(reasoningEffortLevel === "high" + ? { reasoningEffort: "high" as const } + : {}), }, }, } @@ -649,53 +624,44 @@ export const ConfigurationModal = ({ return "Please enter an API key" } - const testModel = - getAllModelOptions(aiAssistantSettings).find( - (m) => m.isTestModel && m.provider === selectedProvider, - )?.value ?? modelsByProvider[selectedProvider][0].value - + const provider = createProvider( + selectedProvider, + apiKey, + aiAssistantSettings, + ) try { - const result = await testApiKey( - apiKey, - testModel, - selectedProvider, - aiAssistantSettings, - ) - if (!result.valid) { - const errorMsg = result.error || "Invalid API key" - setError(errorMsg) - return errorMsg - } - const defaultModels = getAllModelOptions(aiAssistantSettings) - .filter((m) => m.defaultEnabled && m.provider === selectedProvider) - .map((m) => m.value) - if (defaultModels.length > 0) { - setEnabledModels(defaultModels) - } + const listing = await provider.listModels() + setProviderListing(listing) setError(null) void trackEvent(ConsoleEvent.AI_CONFIGURATION_VALIDATE) return true } catch (err) { + const classified = provider.classifyError(err, () => {}) const errorMessage = - err instanceof Error ? err.message : "Failed to validate API key" + classified.type === "invalid_key" + ? "Invalid API key" + : classified.message setError(errorMessage) return errorMessage } - }, [selectedProvider, apiKey, modelsByProvider]) + }, [selectedProvider, apiKey, aiAssistantSettings]) const validateStepTwo = useCallback((): string | boolean => { if (!selectedProvider) return "Please select a provider" - if (enabledModels.length === 0) { + if (effectiveEnabledModels().length === 0) { return "Please enable at least one model" } return true - }, [enabledModels, selectedProvider]) + }, [effectiveEnabledModels, selectedProvider]) const handleStepChange = useCallback( (newStepIndex: number, direction: "next" | "previous") => { // When going back from step 2 to step 1, reset step 2 state but keep API key if (newStepIndex === 0 && direction === "previous") { setEnabledModels([]) + setManualInput("") + setProviderListing(null) + setReasoningEffortLevel("default") setPermissions(DEFAULT_PERMISSIONS) } }, @@ -707,6 +673,9 @@ export const ConfigurationModal = ({ setApiKey("") setError(null) setEnabledModels([]) + setManualInput("") + setProviderListing(null) + setReasoningEffortLevel("default") setPermissions(DEFAULT_PERMISSIONS) }, []) @@ -778,10 +747,14 @@ export const ConfigurationModal = ({ content: ( ), @@ -795,10 +768,11 @@ export const ConfigurationModal = ({ providerName, handleProviderSelect, handleApiKeyChange, + providerListing, enabledModels, + manualInput, + reasoningEffortLevel, permissions, - modelsByProvider, - handleModelToggle, handlePermissionsChange, validateStepOne, validateStepTwo, diff --git a/src/components/SetupAIAssistant/ManageModelsModal.tsx b/src/components/SetupAIAssistant/ManageModelsModal.tsx index 5067af4c8..d1d9c1221 100644 --- a/src/components/SetupAIAssistant/ManageModelsModal.tsx +++ b/src/components/SetupAIAssistant/ManageModelsModal.tsx @@ -1,14 +1,33 @@ -import React, { useState, useCallback, useRef } from "react" +import React, { + useState, + useCallback, + useEffect, + useRef, + useImperativeHandle, + forwardRef, +} from "react" import styled from "styled-components" import * as RadixDialog from "@radix-ui/react-dialog" import { Dialog } from "../Dialog" import { Box } from "../Box" import { Text } from "../Text" import { Button } from "../Button" +import { LoadingSpinner } from "../LoadingSpinner" import { Overlay } from "../Overlay" -import type { CustomProviderDefinition } from "../../utils/ai/settings" +import type { CustomProviderDefinition, ProviderModel } from "../../utils/ai" +import { + BUILTIN_PROVIDERS, + buildListingMetadata, + filterOpenAiChatModels, + formatModelLabel, + getProviderName, + matchesListedModel, + sortModelsNewestFirst, +} from "../../utils/ai" +import { createProviderByType } from "../../utils/ai/registry" import { ModelSettings } from "./ModelSettings" import type { ModelSettingsRef } from "./ModelSettings" +import { ModelPicker } from "./ModelPicker" const ModalContent = styled.div` display: flex; @@ -75,41 +94,232 @@ const ErrorText = styled(Text)` color: ${({ theme }) => theme.color.statusDanger}; ` +const ContentSection = styled(Box).attrs({ + flexDirection: "column", + gap: "2rem", +})` + padding: 2.4rem; + width: 100%; +` + +const LoadingContainer = styled(Box).attrs({ + align: "center", + justifyContent: "center", +})` + width: 100%; + padding: 4rem 0; +` + +export type BuiltinModelsResult = { + enabledModels: string[] + modelLabels: Record + utilityModel?: string + reasoningModels?: string[] +} + +type BuiltinModelsRef = { + getResult: () => BuiltinModelsResult | null + validate: () => string | true +} + +type BuiltinModelsContentProps = { + providerId: string + apiKey: string + enabledModels: string[] + onLoadingChange: (loading: boolean) => void +} + +const BuiltinModelsContent = forwardRef< + BuiltinModelsRef, + BuiltinModelsContentProps +>(({ providerId, apiKey, enabledModels, onLoadingChange }, ref) => { + const [listing, setListing] = useState(null) + const [fetchFailed, setFetchFailed] = useState(false) + const [selectedModels, setSelectedModels] = useState([]) + const [unavailableModels, setUnavailableModels] = useState([]) + const [manualInput, setManualInput] = useState("") + const [isLoading, setIsLoading] = useState(true) + + const isOpenAi = BUILTIN_PROVIDERS[providerId]?.type === "openai" + const pickerModels = listing + ? isOpenAi + ? filterOpenAiChatModels(listing) + : sortModelsNewestFirst(listing) + : [] + const hiddenModels = + listing && isOpenAi + ? sortModelsNewestFirst( + listing.filter((m) => !pickerModels.some((p) => p.id === m.id)), + ) + : undefined + + const selectionWithPending = () => { + const pending = manualInput.trim() + return pending && !selectedModels.includes(pending) + ? [...selectedModels, pending] + : [...selectedModels] + } + + useImperativeHandle( + ref, + () => ({ + getResult: () => { + if (!listing) return null + const models = selectionWithPending() + return { + enabledModels: models, + ...buildListingMetadata(providerId, listing, models), + } + }, + validate: () => { + if (!listing) return "Could not fetch models from the provider" + if (selectionWithPending().length === 0) + return "Enable at least one model" + return true + }, + }), + [listing, selectedModels, manualInput, providerId], + ) + + useEffect(() => { + let cancelled = false + + const doFetch = async () => { + setIsLoading(true) + onLoadingChange(true) + try { + const provider = createProviderByType( + BUILTIN_PROVIDERS[providerId].type, + providerId, + apiKey, + ) + const models = await provider.listModels() + if (cancelled) return + setListing(models) + setSelectedModels( + enabledModels.filter((id) => + models.some((m) => matchesListedModel(id, m.id)), + ), + ) + setUnavailableModels( + enabledModels.filter( + (id) => !models.some((m) => matchesListedModel(id, m.id)), + ), + ) + } catch { + if (cancelled) return + setFetchFailed(true) + } finally { + if (!cancelled) { + setIsLoading(false) + onLoadingChange(false) + } + } + } + + void doFetch() + return () => { + cancelled = true + } + }, []) + + if (isLoading) { + return ( + + + + + + ) + } + + if (fetchFailed) { + return ( + + + Could not fetch models from the provider. Check your API key and + connection, then try again. + + + ) + } + + return ( + + model.label ?? formatModelLabel(model.id)} + onSelectionChange={setSelectedModels} + onManualInputChange={setManualInput} + /> + + ) +}) + +BuiltinModelsContent.displayName = "BuiltinModelsContent" + type ManageModelsModalProps = { open: boolean onOpenChange: (open: boolean) => void providerId: string - definition: CustomProviderDefinition - onSave: (providerId: string, definition: CustomProviderDefinition) => void -} +} & ( + | { + variant: "custom" + definition: CustomProviderDefinition + onSave: (providerId: string, definition: CustomProviderDefinition) => void + } + | { + variant: "builtin" + apiKey: string + enabledModels: string[] + onSave: (providerId: string, result: BuiltinModelsResult) => void + } +) -export const ManageModelsModal = ({ - open, - onOpenChange, - providerId, - definition, - onSave, -}: ManageModelsModalProps) => { +export const ManageModelsModal = (props: ManageModelsModalProps) => { + const { open, onOpenChange, providerId } = props const [error, setError] = useState(null) const [modelsLoading, setModelsLoading] = useState(true) const modelSettingsRef = useRef(null) + const builtinModelsRef = useRef(null) + + const providerName = + props.variant === "custom" + ? props.definition.name + : getProviderName(providerId) const handleSave = useCallback(() => { setError(null) - const result = modelSettingsRef.current?.validate() - if (typeof result === "string") { - setError(result) - return + if (props.variant === "custom") { + const result = modelSettingsRef.current?.validate() + if (typeof result === "string") { + setError(result) + return + } + const values = modelSettingsRef.current?.getValues() + if (!values) return + props.onSave(providerId, { + ...props.definition, + models: values.models, + contextWindow: values.contextWindow, + }) + } else { + const result = builtinModelsRef.current?.validate() + if (typeof result === "string") { + setError(result) + return + } + const values = builtinModelsRef.current?.getResult() + if (!values) return + props.onSave(providerId, values) } - const values = modelSettingsRef.current?.getValues() - if (!values) return - onSave(providerId, { - ...definition, - models: values.models, - contextWindow: values.contextWindow, - }) onOpenChange(false) - }, [definition, providerId, onSave, onOpenChange]) + }, [props, providerId, onOpenChange]) return ( @@ -120,33 +330,43 @@ export const ManageModelsModal = ({ Manage Models - Add or remove models and update the context window for{" "} - {definition.name}. + {props.variant === "custom" + ? `Add or remove models and update the context window for ${providerName}.` + : `Enable the ${providerName} models you want to use.`} - {open && ( + {open && props.variant === "custom" && ( )} + {open && props.variant === "builtin" && ( + + )} diff --git a/src/components/SetupAIAssistant/ModelDropdown.tsx b/src/components/SetupAIAssistant/ModelDropdown.tsx index bcd2478cf..3412d0854 100644 --- a/src/components/SetupAIAssistant/ModelDropdown.tsx +++ b/src/components/SetupAIAssistant/ModelDropdown.tsx @@ -6,7 +6,6 @@ import { useAIStatus } from "../../providers/AIStatusProvider" import { StoreKey } from "../../utils/localStorage/types" import { OpenAIIcon } from "./OpenAIIcon" import { AnthropicIcon } from "./AnthropicIcon" -import { BrainIcon } from "./BrainIcon" import { PlugsIcon, WarningCircleIcon } from "@phosphor-icons/react" import { SelectMenu } from "../SelectMenu" import { trackEvent } from "../../modules/ConsoleEventTracker" @@ -43,7 +42,6 @@ export const ModelDropdown = () => { return null } - // currentModel is guaranteed to be from MODEL_OPTIONS (set in modals) const displayModel = currentModel ? (enabledModels.find((m) => m.value === currentModel) ?? enabledModels[0]) : (enabledModels[0] ?? null) @@ -106,7 +104,6 @@ export const ModelDropdown = () => { > {model.label} - {model.isSlow && } ))} diff --git a/src/components/SetupAIAssistant/ModelPicker.tsx b/src/components/SetupAIAssistant/ModelPicker.tsx new file mode 100644 index 000000000..65734e51a --- /dev/null +++ b/src/components/SetupAIAssistant/ModelPicker.tsx @@ -0,0 +1,332 @@ +import React, { useState } from "react" +import styled, { useTheme } from "styled-components" +import { WarningIcon, XIcon } from "@phosphor-icons/react" +import { Box } from "../Box" +import { Button } from "../Button" +import { Checkbox } from "../Checkbox" +import { IconButton } from "../IconButton" +import { Input } from "../Input" +import { Text } from "../Text" +import { TextButton } from "../TextButton" +import type { ProviderModel } from "../../utils/ai" +import { matchesListedModel, sortModelsNewestFirst } from "../../utils/ai" + +const PickerSection = styled(Box).attrs({ + flexDirection: "column", + gap: "1.2rem", +})` + width: 100%; +` + +const HeaderRow = styled(Box).attrs({ + flexDirection: "row", + gap: "1.2rem", + align: "center", +})` + width: 100%; +` + +const HeaderLabel = styled(Text)` + font-size: 1.6rem; + font-weight: 600; + color: ${({ theme }) => theme.color.contentSecondary}; +` + +const SelectAllRow = styled(Box).attrs({ + gap: "2rem", + align: "center", +})` + display: inline-flex; + margin-left: auto; +` + +const SelectAllLink = styled(TextButton)` + font-size: 1.4rem; +` + +const ModelListContainer = styled.div` + max-height: 30rem; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 0.25rem; + border: 0.1rem solid ${({ theme }) => theme.color.borderStrong}; + border-radius: 0.4rem; + width: 100%; +` + +const ModelRow = styled.label` + display: flex; + align-items: center; + gap: 0.8rem; + padding: 0.6rem 0.8rem; + cursor: pointer; + font-size: 1.4rem; + color: ${({ theme }) => theme.color.contentPrimary}; + + &:hover { + background: ${({ theme }) => theme.color.interactionNeutral}; + } +` + +const ModelIdText = styled(Text)` + font-size: 1.2rem; + color: ${({ theme }) => theme.color.contentSecondary}; +` + +const UnavailableRow = styled(Box).attrs({ + gap: "0.8rem", + align: "center", +})` + padding: 0.6rem 0.8rem; + font-size: 1.4rem; + color: ${({ theme }) => theme.color.contentPrimary}; +` + +const UnavailableHint = styled(Text)` + font-size: 1.2rem; + color: ${({ theme }) => theme.color.statusWarning}; + margin-left: auto; +` + +const ShowAllButton = styled(TextButton)` + font-size: 1.3rem; + align-self: flex-start; +` + +const HelperText = styled(Text)` + font-size: 1.3rem; + font-weight: 400; + color: ${({ theme }) => theme.color.contentSecondary}; +` + +const AddModelRow = styled(Box).attrs({ + gap: "0.8rem", + align: "center", +})` + width: 100%; +` + +const AddModelInput = styled(Input)` + width: 100%; +` + +const AddModelButton = styled(Button).attrs({ variant: "secondary" })` + height: 3rem; + padding: 0 1.2rem; + font-size: 1.4rem; + white-space: nowrap; +` + +const ModelChipsContainer = styled.div` + display: flex; + flex-wrap: wrap; + gap: 0.6rem; +` + +const ModelChip = styled.div` + display: inline-flex; + align-items: center; + gap: 0.5rem; + background: ${({ theme }) => theme.color.interactionNeutral}; + border-radius: 0.4rem; + padding: 0.4rem 0.8rem; + font-size: 1.3rem; + color: ${({ theme }) => theme.color.contentPrimary}; +` + +const ChipRemoveButton = styled(IconButton)` + padding: 0; + width: 2rem; + min-width: 2rem; + height: 2rem; +` + +export type ModelPickerProps = { + listedModels: ProviderModel[] + hiddenModels?: ProviderModel[] + selectedModels: string[] + unavailableModels?: string[] + manualInput: string + dataHookPrefix: string + labelFor?: (model: ProviderModel) => string + onSelectionChange: (models: string[]) => void + onManualInputChange: (value: string) => void +} + +export const ModelPicker = ({ + listedModels, + hiddenModels, + selectedModels, + unavailableModels, + manualInput, + dataHookPrefix, + labelFor, + onSelectionChange, + onManualInputChange, +}: ModelPickerProps) => { + const theme = useTheme() + const [showAll, setShowAll] = useState(false) + + const visibleModels = + showAll && hiddenModels?.length + ? sortModelsNewestFirst([...listedModels, ...hiddenModels]) + : listedModels + const isRowChecked = (rowId: string) => + selectedModels.some((selected) => matchesListedModel(selected, rowId)) + const isListedAnywhere = (selected: string) => + listedModels.some((m) => matchesListedModel(selected, m.id)) || + (hiddenModels?.some((m) => matchesListedModel(selected, m.id)) ?? false) + const manualModels = selectedModels.filter((m) => !isListedAnywhere(m)) + + const handleToggleRow = (rowId: string) => { + if (isRowChecked(rowId)) { + onSelectionChange( + selectedModels.filter((s) => !matchesListedModel(s, rowId)), + ) + } else { + onSelectionChange([...selectedModels, rowId]) + } + } + + const handleSelectAll = () => { + const unchecked = visibleModels + .filter((m) => !isRowChecked(m.id)) + .map((m) => m.id) + onSelectionChange([...selectedModels, ...unchecked]) + } + + const handleDeselectAll = () => { + onSelectionChange( + selectedModels.filter( + (s) => !visibleModels.some((m) => matchesListedModel(s, m.id)), + ), + ) + } + + const handleAddManualModel = () => { + const trimmed = manualInput.trim() + if (!trimmed) return + if (!selectedModels.includes(trimmed)) { + onSelectionChange([...selectedModels, trimmed]) + } + onManualInputChange("") + } + + const handleRemoveManualModel = (model: string) => { + onSelectionChange(selectedModels.filter((m) => m !== model)) + } + + return ( + <> + + + Select Models + + + Select All + + + Deselect All + + + + + {unavailableModels?.map((model) => ( + + + {model} + + Removed by the provider. Save removes it. + + + ))} + {visibleModels.map((model) => { + const label = labelFor ? labelFor(model) : model.id + return ( + + handleToggleRow(model.id)} + /> + {label} + {label !== model.id && {model.id}} + + ) + })} + + {!showAll && !!hiddenModels?.length && ( + setShowAll(true)} + > + Show all models + + )} + + + Don't see your model? Add it manually: + + onManualInputChange(e.target.value)} + placeholder="e.g., llama3, gpt-4o, claude-sonnet-4-20250514" + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault() + handleAddManualModel() + } + }} + /> + + Add + + + {manualModels.length > 0 && ( + + {manualModels.map((model) => ( + + {model} + handleRemoveManualModel(model)} + > + + + + ))} + + )} + + + ) +} diff --git a/src/components/SetupAIAssistant/ModelSettings.tsx b/src/components/SetupAIAssistant/ModelSettings.tsx index 233ac017e..24be0a8c1 100644 --- a/src/components/SetupAIAssistant/ModelSettings.tsx +++ b/src/components/SetupAIAssistant/ModelSettings.tsx @@ -9,15 +9,15 @@ import React, { import styled, { useTheme } from "styled-components" import { Box } from "../Box" import { Input } from "../Input" -import { Checkbox } from "../Checkbox" import { Text } from "../Text" import { LoadingSpinner } from "../LoadingSpinner" import { Button } from "../Button" import { IconButton } from "../IconButton" -import { TextButton } from "../TextButton" import { WarningIcon, XIcon } from "@phosphor-icons/react" import { createProviderByType } from "../../utils/ai/registry" import type { ProviderType } from "../../utils/ai/settings" +import type { ProviderModel } from "../../utils/ai" +import { ModelPicker } from "./ModelPicker" import { PermissionsSection } from "../../scenes/Footer/MCPBridgeStatus/PermissionsSection" import type { Permissions } from "../../utils/tools/permissions" @@ -61,31 +61,6 @@ const WarningText = styled(Text)` color: ${({ theme }) => theme.color.statusWarning}; ` -const ModelListContainer = styled.div` - max-height: 30rem; - overflow-y: auto; - display: flex; - flex-direction: column; - gap: 0.25rem; - border: 0.1rem solid ${({ theme }) => theme.color.borderStrong}; - border-radius: 0.4rem; - width: 100%; -` - -const ModelRow = styled.label` - display: flex; - align-items: center; - gap: 0.8rem; - padding: 0.6rem 0.8rem; - cursor: pointer; - font-size: 1.4rem; - color: ${({ theme }) => theme.color.contentPrimary}; - - &:hover { - background: ${({ theme }) => theme.color.interactionNeutral}; - } -` - const ModelChipsContainer = styled.div` display: flex; flex-wrap: wrap; @@ -124,18 +99,6 @@ const AddModelButton = styled(Button).attrs({ variant: "secondary" })` white-space: nowrap; ` -const SelectAllRow = styled(Box).attrs({ - gap: "2rem", - align: "center", -})` - display: inline-flex; - margin-left: auto; -` - -const SelectAllLink = styled(TextButton)` - font-size: 1.4rem; -` - const ContentSection = styled(Box).attrs({ flexDirection: "column", gap: "2rem", @@ -193,7 +156,7 @@ export type ModelSettingsProps = { async function fetchProviderModels( config: FetchConfig, contextWindow: number, -): Promise { +): Promise { try { const provider = createProviderByType( config.providerType, @@ -215,7 +178,9 @@ export const ModelSettings = forwardRef( ) => { const theme = useTheme() - const [fetchedModels, setFetchedModels] = useState(null) + const [fetchedModels, setFetchedModels] = useState( + null, + ) const [selectedModels, setSelectedModels] = useState([]) const [manualModels, setManualModels] = useState( () => initialValues?.models ?? [], @@ -252,13 +217,8 @@ export const ModelSettings = forwardRef( if (cancelled) return if (models) { - // Auto mode: reconcile initialValues.models against fetched list setFetchedModels(models) - const selected = [ - ...initModels.filter((m) => models.includes(m)), - ...initModels.filter((m) => !models.includes(m)), - ] - setSelectedModels(selected.length > 0 ? selected : []) + setSelectedModels([...initModels]) setManualModels([]) } else { // Manual mode @@ -281,43 +241,14 @@ export const ModelSettings = forwardRef( const isAutoMode = fetchedModels !== null - const handleToggleModel = useCallback((model: string) => { - setSelectedModels((prev) => - prev.includes(model) - ? prev.filter((m) => m !== model) - : [...prev, model], - ) - }, []) - - const handleSelectAll = useCallback(() => { - setSelectedModels((prev) => { - if (!fetchedModels) return prev - const manual = prev.filter((m) => !fetchedModels.includes(m)) - return [...fetchedModels, ...manual] - }) - }, [fetchedModels]) - - const handleDeselectAll = useCallback(() => { - setSelectedModels((prev) => - fetchedModels ? prev.filter((m) => !fetchedModels.includes(m)) : [], - ) - }, [fetchedModels]) - const handleAddManualModel = useCallback(() => { const trimmed = manualModelInput.trim() if (!trimmed) return - - if (isAutoMode) { - setSelectedModels((prev) => - prev.includes(trimmed) ? prev : [...prev, trimmed], - ) - } else { - setManualModels((prev) => - prev.includes(trimmed) ? prev : [...prev, trimmed], - ) - } + setManualModels((prev) => + prev.includes(trimmed) ? prev : [...prev, trimmed], + ) setManualModelInput("") - }, [manualModelInput, isAutoMode]) + }, [manualModelInput]) const handleRemoveManualModel = useCallback((model: string) => { setManualModels((prev) => prev.filter((m) => m !== model)) @@ -397,118 +328,64 @@ export const ModelSettings = forwardRef( )} {isAutoMode && ( - - - Select Models - - - Select All - - - Deselect All - - - - - {fetchedModels.map((model) => ( - - handleToggleModel(model)} - /> - {model} - - ))} - - + )} - - {!isAutoMode && Add Models} - {isAutoMode && ( - - Don't see your model? Add it manually: - - )} - - setManualModelInput(e.target.value)} - placeholder="e.g., llama3, gpt-4o, claude-sonnet-4-20250514" - onKeyDown={(e) => { - if (e.key === "Enter") { - e.preventDefault() - handleAddManualModel() - } - }} - /> - - Add - - - {isAutoMode && - selectedModels.filter((m) => !fetchedModels.includes(m)).length > - 0 && ( + {!isAutoMode && ( + + Add Models + + setManualModelInput(e.target.value)} + placeholder="e.g., llama3, gpt-4o, claude-sonnet-4-20250514" + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault() + handleAddManualModel() + } + }} + /> + + Add + + + {manualModels.length > 0 && ( - {selectedModels - .filter((m) => !fetchedModels.includes(m)) - .map((model) => ( - ( + + {model} + handleRemoveManualModel(model)} > - {model} - handleToggleModel(model)} - > - - - - ))} + + + + ))} )} - {!isAutoMode && manualModels.length > 0 && ( - - {manualModels.map((model) => ( - - {model} - handleRemoveManualModel(model)} - > - - - - ))} - - )} - + + )} AI Assistant uses tools to gather information about QuestDB and your database. Make sure to select the models that support tool calling. diff --git a/src/components/SetupAIAssistant/ReasoningSection.tsx b/src/components/SetupAIAssistant/ReasoningSection.tsx new file mode 100644 index 000000000..c5171760b --- /dev/null +++ b/src/components/SetupAIAssistant/ReasoningSection.tsx @@ -0,0 +1,87 @@ +import React from "react" +import styled from "styled-components" +import { SelectMenu } from "../SelectMenu" + +export type ReasoningEffortLevel = "default" | "high" + +type Option = { + level: ReasoningEffortLevel + label: string + hint: string +} + +const OPTIONS: Option[] = [ + { + level: "default", + label: "Default", + hint: "Each model uses its own default reasoning level.", + }, + { + level: "high", + label: "High", + hint: "Maximum thinking — slower, better for hard questions.", + }, +] + +const Field = styled.div` + display: flex; + flex-direction: column; + gap: 1.6rem; + font-size: 1.1rem; + width: 100%; +` + +const RichTitle = styled.span` + font-size: 1.6rem; + font-weight: 600; + color: ${({ theme }) => theme.color.contentPrimary}; +` + +type Props = { + value: ReasoningEffortLevel + onChange: (next: ReasoningEffortLevel) => void + disabled?: boolean +} + +export const ReasoningSection: React.FC = ({ + value, + onChange, + disabled = false, +}) => { + const current = OPTIONS.find((o) => o.level === value) ?? OPTIONS[0] + + return ( + + Reasoning + + + + + onChange(level as ReasoningEffortLevel)} + > + {OPTIONS.map((opt) => ( + + {opt.label} + + ))} + + + + + + ) +} diff --git a/src/components/SetupAIAssistant/SettingsModal.tsx b/src/components/SetupAIAssistant/SettingsModal.tsx index bb6623644..87cd9db15 100644 --- a/src/components/SetupAIAssistant/SettingsModal.tsx +++ b/src/components/SetupAIAssistant/SettingsModal.tsx @@ -4,21 +4,18 @@ import * as RadixDialog from "@radix-ui/react-dialog" import { Dialog } from "../Dialog" import { Box } from "../Box" import { Input } from "../Input" -import { Switch } from "../Switch" import { Text } from "../Text" import { Button } from "../Button" import { IconButton } from "../IconButton" import { TabButton } from "../TabButton" import { TextButton } from "../TextButton" import { useLocalStorage } from "../../providers/LocalStorageProvider" -import { testApiKey } from "../../utils/ai/aiAssistant" import { StoreKey } from "../../utils/localStorage/types" import { toast } from "../Toast" import { Edit } from "../icons" import { TrashIcon, PlugsIcon, PlusIcon, XIcon } from "@phosphor-icons/react" import { OpenAIIcon } from "./OpenAIIcon" import { AnthropicIcon } from "./AnthropicIcon" -import { BrainIcon } from "./BrainIcon" import { LoadingSpinner } from "../LoadingSpinner" import { Overlay } from "../Overlay" import { @@ -26,12 +23,15 @@ import { getAllModelOptions, getApiKey, makeCustomModelValue, + parseModelValue, + formatModelLabel, BUILTIN_PROVIDERS, type ModelOption, type ProviderId, getNextModel, getProviderName, } from "../../utils/ai" +import { createProvider } from "../../utils/ai/registry" import type { AiAssistantSettings, CustomProviderDefinition, @@ -45,6 +45,9 @@ import { trackEvent } from "../../modules/ConsoleEventTracker" import { ConsoleEvent } from "../../modules/ConsoleEventTracker/events" import { CustomProviderModal } from "./CustomProviderModal" import { ManageModelsModal } from "./ManageModelsModal" +import type { BuiltinModelsResult } from "./ManageModelsModal" +import { ReasoningSection } from "./ReasoningSection" +import type { ReasoningEffortLevel } from "./ReasoningSection" const ModalContent = styled.div` display: flex; @@ -328,13 +331,6 @@ const ModelInfoColumn = styled(Box).attrs({ align-items: flex-start; ` -const ModelInfoRow = styled(Box).attrs({ - gap: "0.8rem", - align: "center", -})` - width: 100%; -` - const ModelDescriptionText = styled(Text)` font-size: 1.1rem; color: ${({ theme }) => theme.color.contentSecondary}; @@ -473,6 +469,40 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { [], ), ) + const [modelLabels, setModelLabels] = useState< + Record> + >(() => + initializeProviderState( + (provider) => + aiAssistantSettings.providers?.[provider]?.modelLabels ?? {}, + {}, + ), + ) + const [utilityModels, setUtilityModels] = useState< + Record + >(() => + initializeProviderState( + (provider) => aiAssistantSettings.providers?.[provider]?.utilityModel, + undefined, + ), + ) + const [reasoningModels, setReasoningModels] = useState< + Record + >(() => + initializeProviderState( + (provider) => aiAssistantSettings.providers?.[provider]?.reasoningModels, + undefined, + ), + ) + const [reasoningEffort, setReasoningEffort] = useState< + Record + >(() => + initializeProviderState( + (provider) => + aiAssistantSettings.providers?.[provider]?.reasoningEffort ?? "default", + "default", + ), + ) const [permissions, setPermissions] = useState< Record >(() => @@ -563,70 +593,38 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { setValidationState((prev) => ({ ...prev, [provider]: "validating" })) setValidationErrors((prev) => ({ ...prev, [provider]: null })) - const providerModels = getModelsForProvider(provider, localSettings) - if (providerModels.length === 0) { - setValidationState((prev) => ({ ...prev, [provider]: "error" })) - setValidationErrors((prev) => ({ - ...prev, - [provider]: "No models available for this provider", - })) - return - } - - const testModel = ( - providerModels.find((m) => m.isTestModel) ?? providerModels[0] - ).value + const isBuiltin = !!BUILTIN_PROVIDERS[provider] try { - const result = await testApiKey( - apiKey, - testModel, - provider, - localSettings, - ) - if (!result.valid) { - setValidationState((prev) => ({ ...prev, [provider]: "error" })) - setValidationErrors((prev) => ({ - ...prev, - [provider]: result.error || "Invalid API key", - })) - } else { - const defaultModels = getAllModelOptions(localSettings) - .filter((m) => m.defaultEnabled && m.provider === provider) - .map((m) => m.value) - if (defaultModels.length > 0) { - setEnabledModels((prev) => ({ ...prev, [provider]: defaultModels })) - } + const aiProvider = createProvider(provider, apiKey, localSettings) + await aiProvider.listModels() + setValidationState((prev) => ({ ...prev, [provider]: "validated" })) + setValidatedApiKeys((prev) => ({ ...prev, [provider]: true })) + setValidationErrors((prev) => ({ ...prev, [provider]: null })) + if (isBuiltin) { + setManageModelsModalOpen(true) + } + } catch (err) { + const aiProvider = createProvider(provider, apiKey, localSettings) + const classified = aiProvider.classifyError(err, () => {}) + if (!isBuiltin && classified.type !== "invalid_key") { + // Custom endpoints often lack a model listing — the key may still work. setValidationState((prev) => ({ ...prev, [provider]: "validated" })) setValidatedApiKeys((prev) => ({ ...prev, [provider]: true })) - setValidationErrors((prev) => ({ ...prev, [provider]: null })) + return } - } catch (err) { setValidationState((prev) => ({ ...prev, [provider]: "error" })) - const errorMessage = - err instanceof Error ? err.message : "Failed to validate API key" - setValidationErrors((prev) => ({ ...prev, [provider]: errorMessage })) + setValidationErrors((prev) => ({ + ...prev, + [provider]: + classified.type === "invalid_key" + ? "Invalid API key" + : classified.message, + })) } }, [apiKeys, localSettings], ) - const handleModelToggle = useCallback( - (provider: ProviderId, modelValue: string) => { - void trackEvent(ConsoleEvent.AI_SETTINGS_MODEL_TOGGLE) - setEnabledModels((prev) => { - const current = prev[provider] - const isEnabled = current.includes(modelValue) - return { - ...prev, - [provider]: isEnabled - ? current.filter((m) => m !== modelValue) - : [...current, modelValue], - } - }) - }, - [], - ) - // Emit the legacy schema-access-removed event on grantSchemaAccess → false // so existing dashboards keep working. const handlePermissionsChange = useCallback( @@ -650,12 +648,25 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { const isCustom = !BUILTIN_PROVIDERS[provider] if (validatedApiKeys[provider] || isCustom) { const perms = permissions[provider] + const labels = modelLabels[provider] updatedProviders[provider] = { apiKey: apiKeys[provider] ?? "", enabledModels: enabledModels[provider], grantSchemaAccess: perms.grantSchemaAccess, read: perms.read, write: perms.write, + ...(labels && Object.keys(labels).length > 0 + ? { modelLabels: labels } + : {}), + ...(utilityModels[provider] + ? { utilityModel: utilityModels[provider] } + : {}), + ...(reasoningModels[provider]?.length + ? { reasoningModels: reasoningModels[provider] } + : {}), + ...(reasoningEffort[provider] === "high" + ? { reasoningEffort: "high" as const } + : {}), } } else { delete updatedProviders[provider] @@ -708,6 +719,10 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { localCustomProviders, apiKeys, enabledModels, + modelLabels, + utilityModels, + reasoningModels, + reasoningEffort, permissions, validatedApiKeys, updateSettings, @@ -741,6 +756,10 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { setValidationState((prev) => ({ ...prev, [providerId]: "idle" })) setValidationErrors((prev) => ({ ...prev, [providerId]: null })) setEnabledModels((prev) => ({ ...prev, [providerId]: [] })) + setModelLabels((prev) => ({ ...prev, [providerId]: {} })) + setUtilityModels((prev) => ({ ...prev, [providerId]: undefined })) + setReasoningModels((prev) => ({ ...prev, [providerId]: undefined })) + setReasoningEffort((prev) => ({ ...prev, [providerId]: "default" })) setIsInputFocused((prev) => ({ ...prev, [providerId]: false })) // Switch to first remaining active provider @@ -887,6 +906,25 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { [aiAssistantSettings, enabledModels, localCustomProviders, updateSettings], ) + const handleBuiltinModelsSave = useCallback( + (providerId: string, result: BuiltinModelsResult) => { + setEnabledModels((prev) => ({ + ...prev, + [providerId]: result.enabledModels, + })) + setModelLabels((prev) => ({ ...prev, [providerId]: result.modelLabels })) + setUtilityModels((prev) => ({ + ...prev, + [providerId]: result.utilityModel, + })) + setReasoningModels((prev) => ({ + ...prev, + [providerId]: result.reasoningModels, + })) + }, + [], + ) + const currentProviderValidated = validatedApiKeys[selectedProvider] const currentProviderApiKey = apiKeys[selectedProvider] const currentProviderValidationState = validationState[selectedProvider] @@ -907,6 +945,11 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { [enabledModels, selectedProvider], ) + const labelForModel = (provider: ProviderId, value: string) => { + if (!BUILTIN_PROVIDERS[provider]) return parseModelValue(value).rawModel + return modelLabels[provider]?.[value] ?? formatModelLabel(value) + } + const allProviders = useMemo( () => getAllProviders(localSettings), [localSettings], @@ -1152,56 +1195,33 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { align="center" style={{ width: "100%" }} > - Enable Models - {isCustomProvider && - (currentProviderValidated || - modelsForProvider.length > 0) && ( - setManageModelsModalOpen(true)} - > - Manage models - - )} + Models + {(currentProviderValidated || + (isCustomProvider && + modelsForProvider.length > 0)) && ( + setManageModelsModalOpen(true)} + > + Manage models + + )} - {currentProviderValidated || - (isCustomProvider && modelsForProvider.length > 0) ? ( + {enabledModelsForProvider.length > 0 ? ( - {modelsForProvider.map((model) => { - const isEnabled = enabledModelsForProvider.includes( - model.value, - ) + {enabledModelsForProvider.map((value) => { + const label = labelForModel(selectedProvider, value) return ( - + - {model.label} - {model.isSlow && ( - - - - Due to advanced reasoning & thinking - capabilities, responses using this model - can be slow. - - + {label} + {!isCustomProvider && label !== value && ( + + {value} + )} - - handleModelToggle( - selectedProvider, - model.value, - ) - } - /> ) })} @@ -1209,14 +1229,28 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { ) : ( - When you've entered and validated your API key, - you'll be able to select and enable available - models. + {currentProviderValidated + ? "No models enabled yet. Use “Manage models” to enable the models you want." + : "When you’ve entered and validated your API key, you’ll be able to enable models via “Manage models”."} )} + {selectedProvider === "openai" && ( + + + setReasoningEffort((prev) => ({ + ...prev, + [selectedProvider]: next, + })) + } + disabled={!currentProviderValidated} + /> + + )} { isCustomProvider && localCustomProviders[selectedProvider] && ( { onSave={handleManageModelsSave} /> )} + {manageModelsModalOpen && !isCustomProvider && ( + + )} ) } diff --git a/src/providers/LocalStorageProvider/types.ts b/src/providers/LocalStorageProvider/types.ts index 7135eafe7..ae57dd944 100644 --- a/src/providers/LocalStorageProvider/types.ts +++ b/src/providers/LocalStorageProvider/types.ts @@ -7,6 +7,10 @@ export type ProviderSettings = { // Optional for back-compat; missing fields default to denied. read?: boolean write?: boolean + modelLabels?: Record + utilityModel?: string + reasoningEffort?: "default" | "high" + reasoningModels?: string[] } export type CustomProviderDefinition = { diff --git a/src/scenes/Footer/MCPBridgeStatus/PermissionsSection.tsx b/src/scenes/Footer/MCPBridgeStatus/PermissionsSection.tsx index 3c4bb728a..d4194472e 100644 --- a/src/scenes/Footer/MCPBridgeStatus/PermissionsSection.tsx +++ b/src/scenes/Footer/MCPBridgeStatus/PermissionsSection.tsx @@ -62,8 +62,12 @@ const FieldLabel = styled.span` font-weight: 600; ` +const RichField = styled(Field)` + gap: 1.6rem; +` + const RichTitle = styled.span` - font-size: 1.8rem; + font-size: 1.6rem; font-weight: 600; color: ${({ theme }) => theme.color.contentPrimary}; ` @@ -123,13 +127,13 @@ export const PermissionsSection: React.FC = ({ if (variant === "rich") { return ( - + Permissions {trigger} {content} - + ) } diff --git a/src/utils/ai/aiAssistant.ts b/src/utils/ai/aiAssistant.ts index 157640543..89f6ee398 100644 --- a/src/utils/ai/aiAssistant.ts +++ b/src/utils/ai/aiAssistant.ts @@ -350,16 +350,6 @@ const tryWithRetries = async ( } } -export const testApiKey = async ( - apiKey: string, - model: string, - providerId: ProviderId, - settings?: AiAssistantSettings, -): Promise<{ valid: boolean; error?: string }> => { - const provider = createProvider(providerId, apiKey, settings) - return provider.testConnection({ apiKey, model }) -} - export const generateChatTitle = async ({ firstUserMessage, settings, diff --git a/src/utils/ai/anthropicProvider.ts b/src/utils/ai/anthropicProvider.ts index 5a2731f55..b1677a8f9 100644 --- a/src/utils/ai/anthropicProvider.ts +++ b/src/utils/ai/anthropicProvider.ts @@ -8,12 +8,13 @@ import type { StreamingCallback, TokenUsage, } from "./aiAssistant" -import { getModelProps } from "./settings" +import { parseModelValue } from "./settings" import type { ProviderId } from "./settings" import { type AIProvider, type ExecuteFlowParams, type FlowResult, + type ProviderModel, type ToolDefinition, type Message, } from "./types" @@ -130,7 +131,7 @@ function toAnthropicTools(tools: ToolDefinition[]): AnthropicTool[] { } function toAnthropicModel(model: string): string { - return getModelProps(model).model + return parseModelValue(model).rawModel } async function createAnthropicMessage( @@ -397,7 +398,6 @@ async function handleToolCalls( system: systemPrompt, ...(!isLastRound && { tools }), messages: updatedHistory, - temperature: 0.3, } const followUpMessage = streaming @@ -507,7 +507,6 @@ export function createAnthropicProvider( system: systemPrompt, tools: anthropicTools, messages: initialMessages, - temperature: 0.3, } const message = streaming @@ -587,7 +586,6 @@ export function createAnthropicProvider( model: toAnthropicModel(model), messages: [{ role: "user", content: prompt }], max_tokens: 100, - temperature: 0.3, }) const textBlock = message.content.find((block) => block.type === "text") @@ -611,7 +609,7 @@ export function createAnthropicProvider( let text = "" const stream = anthropic.messages.stream( { - ...getModelProps(model), + model: toAnthropicModel(model), max_tokens: 64_000, messages: [{ role: "user", content: userMessage }], system: systemPrompt, @@ -629,54 +627,6 @@ export function createAnthropicProvider( return text }, - async testConnection({ apiKey: testApiKey, model }) { - try { - const testClient = new Anthropic({ - apiKey: testApiKey, - dangerouslyAllowBrowser: true, - ...(options?.baseURL ? { baseURL: options.baseURL } : {}), - ...(isCustom - ? { - fetch: createHeaderFilteredFetch(ANTHROPIC_ALLOWED_HEADERS), - } - : {}), - }) - - await createAnthropicMessage(testClient, { - model: toAnthropicModel(model), - messages: [{ role: "user", content: "ping" }], - max_tokens: 16, - }) - return { valid: true } - } catch (error: unknown) { - if (error instanceof MaxTokensError || error instanceof RefusalError) { - return { valid: true } - } - if (error instanceof Anthropic.AuthenticationError) { - return { valid: false, error: "Invalid API key" } - } - if (error instanceof Anthropic.RateLimitError) { - return { valid: true } - } - const status = - (error as { status?: number })?.status || - (error as { error?: { status?: number } })?.error?.status - if (status === 401) { - return { valid: false, error: "Invalid API key" } - } - if (status === 429) { - return { valid: true } - } - return { - valid: false, - error: - error instanceof Error - ? error.message - : "Failed to validate API key", - } - } - }, - async countTokens({ messages, systemPrompt, model }) { // Custom providers (non-default baseURL) use chars/3.5 estimation // because the actual tokenizer is unknown and most custom endpoints @@ -697,12 +647,16 @@ export function createAnthropicProvider( return response.input_tokens }, - async listModels(): Promise { - const models: string[] = [] + async listModels(): Promise { + const models: ProviderModel[] = [] for await (const model of anthropic.models.list()) { - models.push(model.id) + models.push({ + id: model.id, + label: model.display_name, + created: Math.floor(Date.parse(model.created_at) / 1000), + }) } - return models.sort((a, b) => a.localeCompare(b)) + return models.sort((a, b) => a.id.localeCompare(b.id)) }, classifyError( diff --git a/src/utils/ai/contextCompaction.ts b/src/utils/ai/contextCompaction.ts index 31761ad56..b9bf202ff 100644 --- a/src/utils/ai/contextCompaction.ts +++ b/src/utils/ai/contextCompaction.ts @@ -1,5 +1,5 @@ import type { ConversationMessage } from "../../providers/AIConversationProvider/types" -import { getTestModel } from "./index" +import { getSelectedModel } from "./index" import type { AIProvider, Message } from "./index" import { getMessageTextLength } from "./shared" import type { AiAssistantSettings } from "../../providers/LocalStorageProvider/types" @@ -89,9 +89,9 @@ async function generateSummary( settings?: AiAssistantSettings, abortSignal?: AbortSignal, ): Promise { - const testModelValue = getTestModel(aiProvider.id, settings) - if (!testModelValue) { - throw new Error("No test model found for provider") + const summaryModel = settings ? getSelectedModel(settings) : null + if (!summaryModel) { + throw new Error("No model selected for summarization") } const conversationText = middleMessages @@ -123,7 +123,7 @@ async function generateSummary( const userMessage = `Please summarize the following conversation:\n\n${conversationText}` return aiProvider.generateSummary({ - model: testModelValue, + model: summaryModel, systemPrompt: SUMMARIZATION_PROMPT, userMessage, abortSignal, diff --git a/src/utils/ai/executeAIFlow.ts b/src/utils/ai/executeAIFlow.ts index 26c69f188..e16f90204 100644 --- a/src/utils/ai/executeAIFlow.ts +++ b/src/utils/ai/executeAIFlow.ts @@ -33,7 +33,7 @@ import { type StreamingCallback, } from "./aiAssistant" import { getExplainSchemaPrompt, getHealthIssuePrompt } from "./index" -import { providerForModel, getTestModel, getAllModelOptions } from "./index" +import { providerForModel, getUtilityModel, getAllModelOptions } from "./index" import type { AiAssistantSettings } from "../../providers/LocalStorageProvider/types" import { eventBus } from "../../modules/EventBus" import { EventType } from "../../modules/EventBus/types" @@ -415,14 +415,14 @@ async function generateChatTitleIfNeeded( ) if (!provider) return - const testModelValue = getTestModel(provider, config.aiAssistantSettings) - if (!testModelValue) return + const utilityModel = getUtilityModel(provider, config.aiAssistantSettings) + if (!utilityModel) return try { const title = await generateChatTitle({ firstUserMessage: userMessageContent, settings: { - model: testModelValue, + model: utilityModel, provider, apiKey: config.settings.apiKey, aiAssistantSettings: config.aiAssistantSettings, diff --git a/src/utils/ai/index.ts b/src/utils/ai/index.ts index f587f1556..234c11d1d 100644 --- a/src/utils/ai/index.ts +++ b/src/utils/ai/index.ts @@ -24,17 +24,17 @@ export { } from "./prompts" export type { HealthIssuePromptData } from "./prompts" export { - MODEL_OPTIONS, BUILTIN_PROVIDERS, + buildListingMetadata, providerForModel, - getModelProps, getProviderName, getAllProviders, getAllModelOptions, getAllEnabledModels, + getModelLabel, getSelectedModel, getNextModel, - getTestModel, + getUtilityModel, getProviderContextWindow, getApiKey, makeCustomModelValue, @@ -51,3 +51,13 @@ export type { ModelOption, CustomProviderDefinition, } from "./settings" +export { + computeReasoningModels, + filterOpenAiChatModels, + formatModelLabel, + matchesListedModel, + resolveUtilityModel, + sortModelsNewestFirst, + UTILITY_MODEL_TIERS, +} from "./modelCatalog" +export type { ProviderModel } from "./modelCatalog" diff --git a/src/utils/ai/modelCatalog.test.ts b/src/utils/ai/modelCatalog.test.ts new file mode 100644 index 000000000..ed59a8821 --- /dev/null +++ b/src/utils/ai/modelCatalog.test.ts @@ -0,0 +1,167 @@ +import { describe, it, expect } from "vitest" +import { + computeReasoningModels, + filterOpenAiChatModels, + formatModelLabel, + isReasoningModel, + matchesListedModel, + resolveUtilityModel, + stripDateSuffix, + UTILITY_MODEL_TIERS, +} from "./modelCatalog" +import type { ProviderModel } from "./modelCatalog" + +const model = (id: string, created?: number): ProviderModel => ({ id, created }) + +const AUG_2025 = Date.UTC(2025, 7, 7) / 1000 +const JUL_2025 = Date.UTC(2025, 6, 1) / 1000 + +describe("stripDateSuffix", () => { + it("strips Anthropic and OpenAI date suffixes", () => { + expect(stripDateSuffix("claude-sonnet-4-5-20250929")).toBe( + "claude-sonnet-4-5", + ) + expect(stripDateSuffix("gpt-4.1-2025-04-14")).toBe("gpt-4.1") + expect(stripDateSuffix("gpt-4-0613")).toBe("gpt-4") + }) + + it("keeps non-date suffixes", () => { + expect(stripDateSuffix("gpt-3.5-turbo-16k")).toBe("gpt-3.5-turbo-16k") + expect(stripDateSuffix("claude-sonnet-4-5")).toBe("claude-sonnet-4-5") + }) +}) + +describe("matchesListedModel", () => { + it("matches exact ids", () => { + expect(matchesListedModel("gpt-5.4", "gpt-5.4")).toBe(true) + }) + + it("matches a stored alias against its dated listing id", () => { + expect( + matchesListedModel("claude-sonnet-4-5", "claude-sonnet-4-5-20250929"), + ).toBe(true) + }) + + it("does not match a stored dated id against a different dated id", () => { + expect(matchesListedModel("gpt-4-0613", "gpt-4-1106")).toBe(false) + }) + + it("does not match unrelated ids", () => { + expect(matchesListedModel("gpt-5.4", "gpt-5.4-mini")).toBe(false) + }) +}) + +describe("formatModelLabel", () => { + it("derives labels from OpenAI ids", () => { + expect(formatModelLabel("gpt-5-mini")).toBe("GPT-5 Mini") + expect(formatModelLabel("gpt-5.4")).toBe("GPT-5.4") + expect(formatModelLabel("gpt-5.6-luna")).toBe("GPT-5.6 Luna") + expect(formatModelLabel("gpt-4o")).toBe("GPT-4o") + expect(formatModelLabel("o4-mini")).toBe("o4 Mini") + }) + + it("joins consecutive version numbers with dots", () => { + expect(formatModelLabel("claude-sonnet-4-5")).toBe("Claude Sonnet 4.5") + }) + + it("drops the date suffix", () => { + expect(formatModelLabel("claude-opus-4-5-20251101")).toBe("Claude Opus 4.5") + expect(formatModelLabel("gpt-5.4-nano-2026-03-17")).toBe("GPT-5.4 Nano") + }) +}) + +describe("filterOpenAiChatModels", () => { + it("drops known non-chat models and dated snapshots", () => { + // Given a listing with chat models, noise, and dated snapshots + const listing = [ + model("gpt-5.4", 300), + model("gpt-5.4-2026-03-05", 300), + model("text-embedding-3-small", 1), + model("whisper-1", 1), + model("gpt-4o-mini-tts", 1), + model("gpt-5-chat-latest", 200), + model("gpt-5.3-codex", 250), + model("sora-2", 250), + model("davinci-002", 1), + ] + // When the filter runs + const kept = filterOpenAiChatModels(listing).map((m) => m.id) + // Then only the plain chat model remains + expect(kept).toEqual(["gpt-5.4"]) + }) + + it("keeps a brand-new generation without any code change", () => { + const kept = filterOpenAiChatModels([model("gpt-6", 500)]).map((m) => m.id) + expect(kept).toEqual(["gpt-6"]) + }) + + it("sorts newest first", () => { + const kept = filterOpenAiChatModels([ + model("gpt-4.1", 100), + model("gpt-5.4", 300), + model("gpt-5", 200), + ]).map((m) => m.id) + expect(kept).toEqual(["gpt-5.4", "gpt-5", "gpt-4.1"]) + }) +}) + +describe("resolveUtilityModel", () => { + it("picks the newest model of the highest-priority tier", () => { + // Given luna and nano models where nano is newer + const listing = [ + model("gpt-5.6-luna", 300), + model("gpt-5.7-nano", 400), + model("gpt-5.4-mini", 200), + ] + // When resolving with OpenAI tiers + const utility = resolveUtilityModel(listing, UTILITY_MODEL_TIERS.openai) + // Then priority beats recency + expect(utility).toBe("gpt-5.6-luna") + }) + + it("falls through to lower tiers when the top tier is absent", () => { + const listing = [model("gpt-5.4-mini", 200), model("gpt-5.4-nano", 200)] + expect(resolveUtilityModel(listing, UTILITY_MODEL_TIERS.openai)).toBe( + "gpt-5.4-nano", + ) + }) + + it("collapses dated ids to their alias and keeps the newest", () => { + const listing = [ + model("claude-haiku-4-5-20251001", 100), + model("claude-haiku-4-6", 200), + model("claude-sonnet-5", 300), + ] + expect(resolveUtilityModel(listing, UTILITY_MODEL_TIERS.anthropic)).toBe( + "claude-haiku-4-6", + ) + }) + + it("returns null when no tier matches", () => { + expect( + resolveUtilityModel([model("gpt-5.4", 100)], UTILITY_MODEL_TIERS.openai), + ).toBeNull() + }) +}) + +describe("reasoning gate", () => { + it("gates models created on or after the gpt-5 launch", () => { + const listing = [ + model("gpt-5", AUG_2025), + model("gpt-4.1", JUL_2025), + model("gpt-6", AUG_2025 + 1_000_000), + ] + expect(computeReasoningModels(listing)).toEqual(["gpt-5", "gpt-6"]) + }) + + it("treats models without a timestamp as ungated", () => { + expect(computeReasoningModels([{ id: "mystery-model" }])).toEqual([]) + }) + + it("matches enabled aliases against gated dated ids", () => { + const gated = ["gpt-5.4-2026-03-05", "gpt-5.4"] + expect(isReasoningModel("gpt-5.4", gated)).toBe(true) + expect(isReasoningModel("gpt-4.1", gated)).toBe(false) + expect(isReasoningModel("gpt-4.1", undefined)).toBe(false) + }) +}) diff --git a/src/utils/ai/modelCatalog.ts b/src/utils/ai/modelCatalog.ts new file mode 100644 index 000000000..86e4cc113 --- /dev/null +++ b/src/utils/ai/modelCatalog.ts @@ -0,0 +1,119 @@ +export type ProviderModel = { + id: string + label?: string + created?: number +} + +const DATE_SUFFIX = /-(\d{8}|\d{4}-\d{2}-\d{2}|\d{4})$/ + +const OPENAI_NON_CHAT_TOKENS = [ + "embedding", + "tts", + "whisper", + "audio", + "realtime", + "image", + "dall-e", + "sora", + "transcribe", + "transcription", + "moderation", + "search", + "codex", + "computer-use", + "chat-latest", + "chatgpt", + "babbage", + "davinci", + "instruct", +] + +export const stripDateSuffix = (id: string): string => + id.replace(DATE_SUFFIX, "") + +export const matchesListedModel = ( + storedId: string, + listedId: string, +): boolean => storedId === listedId || stripDateSuffix(listedId) === storedId + +const isNumericToken = (token: string): boolean => /^[\d.]+$/.test(token) + +export const formatModelLabel = (id: string): string => { + const tokens = stripDateSuffix(id).split("-") + const parts: string[] = [] + for (const token of tokens) { + if (token.toLowerCase() === "gpt") { + parts.push("GPT") + continue + } + const previous = parts[parts.length - 1] + if (/\d/.test(token)) { + if (previous === "GPT") { + parts[parts.length - 1] = `GPT-${token}` + } else if (previous && isNumericToken(previous)) { + parts[parts.length - 1] = `${previous}.${token}` + } else { + parts.push(token) + } + continue + } + parts.push(token.charAt(0).toUpperCase() + token.slice(1)) + } + return parts.join(" ") || id +} + +export const sortModelsNewestFirst = ( + models: ProviderModel[], +): ProviderModel[] => + [...models].sort( + (a, b) => (b.created ?? 0) - (a.created ?? 0) || a.id.localeCompare(b.id), + ) + +const isOpenAiNonChatModel = (id: string): boolean => + OPENAI_NON_CHAT_TOKENS.some((token) => id.includes(token)) || + DATE_SUFFIX.test(id) + +export const filterOpenAiChatModels = ( + models: ProviderModel[], +): ProviderModel[] => + sortModelsNewestFirst(models.filter((m) => !isOpenAiNonChatModel(m.id))) + +export const UTILITY_MODEL_TIERS: Record<"anthropic" | "openai", string[]> = { + anthropic: ["haiku", "sonnet"], + openai: ["luna", "nano", "mini"], +} + +export const resolveUtilityModel = ( + models: ProviderModel[], + tiers: string[], +): string | null => { + const byAlias = new Map() + for (const model of models) { + const alias = stripDateSuffix(model.id) + const existing = byAlias.get(alias) + if (!existing || (model.created ?? 0) > (existing.created ?? 0)) { + byAlias.set(alias, { ...model, id: alias }) + } + } + for (const tier of tiers) { + const matches = sortModelsNewestFirst( + [...byAlias.values()].filter((m) => m.id.includes(tier)), + ) + if (matches.length > 0) return matches[0].id + } + return null +} + +const REASONING_GATE_START = Date.UTC(2025, 7, 1) / 1000 + +export const computeReasoningModels = (models: ProviderModel[]): string[] => + models + .filter((m) => (m.created ?? 0) >= REASONING_GATE_START) + .map((m) => m.id) + +export const isReasoningModel = ( + modelId: string, + reasoningModels: string[] | undefined, +): boolean => + reasoningModels?.some((listed) => matchesListedModel(modelId, listed)) ?? + false diff --git a/src/utils/ai/openaiChatCompletionsProvider.ts b/src/utils/ai/openaiChatCompletionsProvider.ts index b4db2775e..47a7c1738 100644 --- a/src/utils/ai/openaiChatCompletionsProvider.ts +++ b/src/utils/ai/openaiChatCompletionsProvider.ts @@ -9,12 +9,14 @@ import type { StreamingCallback, TokenUsage, } from "./aiAssistant" -import { getModelProps } from "./settings" +import { parseModelValue } from "./settings" import type { ProviderId } from "./settings" +import { isReasoningModel } from "./modelCatalog" import { type AIProvider, type ExecuteFlowParams, type FlowResult, + type ProviderModel, type ToolDefinition, type Message, } from "./types" @@ -34,6 +36,7 @@ import { classifyOpenAIError, countTokensFromNativePayload, isOpenAINonRetryableError, + isReasoningRejection, } from "./openaiShared" import { createHeaderFilteredFetch, @@ -288,6 +291,28 @@ async function executeRequest( params: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming, streaming?: StreamingCallback, abortSignal?: AbortSignal, +): Promise { + try { + return await executeRequestOnce(openai, params, streaming, abortSignal) + } catch (error) { + if (params.reasoning_effort && isReasoningRejection(error)) { + const { reasoning_effort: _reasoningEffort, ...withoutReasoning } = params + return executeRequestOnce( + openai, + withoutReasoning, + streaming, + abortSignal, + ) + } + throw error + } +} + +async function executeRequestOnce( + openai: OpenAI, + params: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming, + streaming?: StreamingCallback, + abortSignal?: AbortSignal, ): Promise { if (streaming) { const accumulated = await createChatCompletionStreaming( @@ -350,23 +375,15 @@ async function executeRequest( } } -function toChatCompletionsAPIProps(model: string): { - model: string - reasoning_effort?: OpenAI.ReasoningEffort -} { - const props = getModelProps(model) - return { - model: props.model, - ...(props.reasoningEffort - ? { reasoning_effort: props.reasoningEffort as OpenAI.ReasoningEffort } - : {}), - } -} - export function createOpenAIChatCompletionsProvider( apiKey: string, providerId: ProviderId = "openai", - options?: { baseURL?: string; contextWindow?: number; isCustom?: boolean }, + options?: { + baseURL?: string + contextWindow?: number + isCustom?: boolean + reasoning?: { effort: "high"; models: string[] } + }, ): AIProvider { const isCustom = options?.isCustom ?? false const openai = new OpenAI({ @@ -382,6 +399,19 @@ export function createOpenAIChatCompletionsProvider( const contextWindow = options?.contextWindow ?? 400_000 + const toRequestProps = ( + model: string, + ): { model: string; reasoning_effort?: OpenAI.ReasoningEffort } => { + const rawModel = parseModelValue(model).rawModel + return { + model: rawModel, + ...(options?.reasoning && + isReasoningModel(rawModel, options.reasoning.models) + ? { reasoning_effort: options.reasoning.effort } + : {}), + } + } + return { id: providerId, contextWindow, @@ -421,7 +451,7 @@ export function createOpenAIChatCompletionsProvider( const toolContext: ToolExecutionContext = incomingToolContext ?? {} const baseParams = { - ...toChatCompletionsAPIProps(model), + ...toRequestProps(model), tools: openaiTools, } @@ -563,7 +593,7 @@ export function createOpenAIChatCompletionsProvider( async generateTitle({ model, prompt }) { try { const response = await openai.chat.completions.create({ - model: toChatCompletionsAPIProps(model).model, + model: parseModelValue(model).rawModel, messages: [{ role: "user", content: prompt }], ...(isCustom ? {} : { max_completion_tokens: 100 }), }) @@ -585,17 +615,34 @@ export function createOpenAIChatCompletionsProvider( abortSignal?: AbortSignal }) { let text = "" - const stream = await openai.chat.completions.create( - { - ...toChatCompletionsAPIProps(model), - messages: [ - { role: "system", content: systemPrompt }, - { role: "user", content: userMessage }, - ], - stream: true, - }, - ...(abortSignal ? [{ signal: abortSignal }] : ([] as const)), - ) + const summaryParams = { + ...toRequestProps(model), + messages: [ + { role: "system" as const, content: systemPrompt }, + { role: "user" as const, content: userMessage }, + ], + stream: true as const, + } + const requestOptions = abortSignal + ? ([{ signal: abortSignal }] as const) + : ([] as const) + let stream + try { + stream = await openai.chat.completions.create( + summaryParams, + ...requestOptions, + ) + } catch (error) { + if (!summaryParams.reasoning_effort || !isReasoningRejection(error)) { + throw error + } + const { reasoning_effort: _reasoningEffort, ...withoutReasoning } = + summaryParams + stream = await openai.chat.completions.create( + withoutReasoning, + ...requestOptions, + ) + } for await (const chunk of stream) { const delta = chunk.choices[0]?.delta?.content if (delta) { @@ -605,43 +652,6 @@ export function createOpenAIChatCompletionsProvider( return text }, - async testConnection({ apiKey: testApiKey, model }) { - try { - const testClient = new OpenAI({ - apiKey: testApiKey, - dangerouslyAllowBrowser: true, - ...(options?.baseURL ? { baseURL: options.baseURL } : {}), - ...(isCustom - ? { - fetch: createHeaderFilteredFetch(OPENAI_ALLOWED_HEADERS), - } - : {}), - }) - await testClient.chat.completions.create({ - model: getModelProps(model).model, - messages: [{ role: "user", content: "ping" }], - }) - return { valid: true } - } catch (error: unknown) { - const status = - (error as { status?: number })?.status || - (error as { error?: { status?: number } })?.error?.status - if (status === 401) { - return { valid: false, error: "Invalid API key" } - } - if (status === 429) { - return { valid: true } - } - return { - valid: false, - error: - error instanceof Error - ? error.message - : "Failed to validate API key", - } - } - }, - async countTokens({ messages, systemPrompt }) { // Custom providers (non-default baseURL) use chars/3.5 estimation if (options?.baseURL) { @@ -655,12 +665,17 @@ export function createOpenAIChatCompletionsProvider( return countTokensFromNativePayload(systemPrompt, nativeMessages) }, - async listModels(): Promise { - const models: string[] = [] + async listModels(): Promise { + const models: ProviderModel[] = [] for await (const model of openai.models.list()) { - models.push(model.id) + const name = (model as { name?: unknown }).name + models.push({ + id: model.id, + created: model.created, + ...(typeof name === "string" ? { label: name } : {}), + }) } - return models.sort((a, b) => a.localeCompare(b)) + return models.sort((a, b) => a.id.localeCompare(b.id)) }, classifyError( diff --git a/src/utils/ai/openaiProvider.ts b/src/utils/ai/openaiProvider.ts index 14dac9d68..57f0ecd4b 100644 --- a/src/utils/ai/openaiProvider.ts +++ b/src/utils/ai/openaiProvider.ts @@ -5,12 +5,14 @@ import type { StatusCallback, StreamingCallback, } from "./aiAssistant" -import { getModelProps } from "./settings" +import { parseModelValue } from "./settings" import type { ProviderId } from "./settings" +import { isReasoningModel } from "./modelCatalog" import { type AIProvider, type ExecuteFlowParams, type FlowResult, + type ProviderModel, type ToolDefinition, type Message, } from "./types" @@ -28,6 +30,7 @@ import { classifyOpenAIError, countTokensFromNativePayload, isOpenAINonRetryableError, + isReasoningRejection, } from "./openaiShared" import { createHeaderFilteredFetch, @@ -255,28 +258,37 @@ function getOpenAIText(response: OpenAI.Responses.Response): { return { type: "text", message: "" } } -function toResponsesAPIProps(model: string): { - model: string - reasoning?: OpenAI.Reasoning -} { - const props = getModelProps(model) - return { - model: props.model, - ...(props.reasoningEffort - ? { - reasoning: { - effort: props.reasoningEffort as OpenAI.ReasoningEffort, - summary: "auto", - }, - } - : {}), +async function createResponseWithReasoningFallback( + openai: OpenAI, + params: OpenAI.Responses.ResponseCreateParamsNonStreaming, + streaming?: StreamingCallback, + abortSignal?: AbortSignal, +): Promise { + const run = async (p: OpenAI.Responses.ResponseCreateParamsNonStreaming) => + streaming + ? (await createOpenAIResponseStreaming(openai, p, streaming, abortSignal)) + .response + : openai.responses.create(p) + try { + return await run(params) + } catch (error) { + if (params.reasoning && isReasoningRejection(error)) { + const { reasoning: _reasoning, ...withoutReasoning } = params + return run(withoutReasoning) + } + throw error } } export function createOpenAIProvider( apiKey: string, providerId: ProviderId = "openai", - options?: { baseURL?: string; contextWindow?: number; isCustom?: boolean }, + options?: { + baseURL?: string + contextWindow?: number + isCustom?: boolean + reasoning?: { effort: "high"; models: string[] } + }, ): AIProvider { const isCustom = options?.isCustom ?? false const openai = new OpenAI({ @@ -292,6 +304,24 @@ export function createOpenAIProvider( const contextWindow = options?.contextWindow ?? 400_000 + const toRequestProps = ( + model: string, + ): { model: string; reasoning?: OpenAI.Reasoning } => { + const rawModel = parseModelValue(model).rawModel + return { + model: rawModel, + ...(options?.reasoning && + isReasoningModel(rawModel, options.reasoning.models) + ? { + reasoning: { + effort: options.reasoning.effort, + summary: "auto" as const, + }, + } + : {}), + } + } + return { id: providerId, contextWindow, @@ -329,7 +359,7 @@ export function createOpenAIProvider( const toolContext: ToolExecutionContext = incomingToolContext ?? {} const requestParams = { - ...toResponsesAPIProps(model), + ...toRequestProps(model), instructions: config.systemInstructions, input, tools: openaiTools, @@ -337,17 +367,12 @@ export function createOpenAIProvider( include: ["reasoning.encrypted_content"], } as OpenAI.Responses.ResponseCreateParamsNonStreaming - const streamResult = streaming - ? await createOpenAIResponseStreaming( - openai, - requestParams, - streaming, - abortSignal, - ) - : { - response: await openai.responses.create(requestParams), - } - let lastResponse = streamResult.response + let lastResponse = await createResponseWithReasoningFallback( + openai, + requestParams, + streaming, + abortSignal, + ) input = [...input, ...lastResponse.output] totalInputTokens += lastResponse.usage?.input_tokens ?? 0 @@ -437,7 +462,7 @@ export function createOpenAIProvider( streaming?.onResponseStart?.() const loopRequestParams = { - ...toResponsesAPIProps(model), + ...toRequestProps(model), instructions: config.systemInstructions, input, ...(!isLastRound && { tools: openaiTools }), @@ -445,17 +470,12 @@ export function createOpenAIProvider( include: ["reasoning.encrypted_content"], } as OpenAI.Responses.ResponseCreateParamsNonStreaming - const loopResult = streaming - ? await createOpenAIResponseStreaming( - openai, - loopRequestParams, - streaming, - abortSignal, - ) - : { - response: await openai.responses.create(loopRequestParams), - } - lastResponse = loopResult.response + lastResponse = await createResponseWithReasoningFallback( + openai, + loopRequestParams, + streaming, + abortSignal, + ) input = [...input, ...lastResponse.output] totalInputTokens += lastResponse.usage?.input_tokens ?? 0 @@ -502,7 +522,7 @@ export function createOpenAIProvider( async generateTitle({ model, prompt }) { try { const response = await openai.responses.create({ - model: toResponsesAPIProps(model).model, + model: parseModelValue(model).rawModel, input: [{ role: "user", content: prompt }], max_output_tokens: 100, }) @@ -524,15 +544,28 @@ export function createOpenAIProvider( abortSignal?: AbortSignal }) { let text = "" - const stream = await openai.responses.create( - { - ...toResponsesAPIProps(model), - instructions: systemPrompt, - input: userMessage, - stream: true, - }, - ...(abortSignal ? [{ signal: abortSignal }] : ([] as const)), - ) + const summaryParams = { + ...toRequestProps(model), + instructions: systemPrompt, + input: userMessage, + stream: true as const, + } + const requestOptions = abortSignal + ? ([{ signal: abortSignal }] as const) + : ([] as const) + let stream + try { + stream = await openai.responses.create(summaryParams, ...requestOptions) + } catch (error) { + if (!summaryParams.reasoning || !isReasoningRejection(error)) { + throw error + } + const { reasoning: _reasoning, ...withoutReasoning } = summaryParams + stream = await openai.responses.create( + withoutReasoning, + ...requestOptions, + ) + } for await (const event of stream) { if (event.type === "response.output_text.delta" && "delta" in event) { text += event.delta @@ -541,43 +574,6 @@ export function createOpenAIProvider( return text }, - async testConnection({ apiKey: testApiKey, model }) { - try { - const testClient = new OpenAI({ - apiKey: testApiKey, - dangerouslyAllowBrowser: true, - ...(options?.baseURL ? { baseURL: options.baseURL } : {}), - ...(isCustom - ? { - fetch: createHeaderFilteredFetch(OPENAI_ALLOWED_HEADERS), - } - : {}), - }) - await testClient.responses.create({ - model: getModelProps(model).model, - input: [{ role: "user", content: "ping" }], - }) - return { valid: true } - } catch (error: unknown) { - const status = - (error as { status?: number })?.status || - (error as { error?: { status?: number } })?.error?.status - if (status === 401) { - return { valid: false, error: "Invalid API key" } - } - if (status === 429) { - return { valid: true } - } - return { - valid: false, - error: - error instanceof Error - ? error.message - : "Failed to validate API key", - } - } - }, - async countTokens({ messages, systemPrompt }) { // Custom providers (non-default baseURL) use chars/3.5 estimation if (options?.baseURL) { @@ -591,12 +587,17 @@ export function createOpenAIProvider( return countTokensFromNativePayload(systemPrompt, nativeInput) }, - async listModels(): Promise { - const models: string[] = [] + async listModels(): Promise { + const models: ProviderModel[] = [] for await (const model of openai.models.list()) { - models.push(model.id) + const name = (model as { name?: unknown }).name + models.push({ + id: model.id, + created: model.created, + ...(typeof name === "string" ? { label: name } : {}), + }) } - return models.sort((a, b) => a.localeCompare(b)) + return models.sort((a, b) => a.id.localeCompare(b.id)) }, classifyError( diff --git a/src/utils/ai/openaiShared.ts b/src/utils/ai/openaiShared.ts index e6209e2bd..f99b55a5e 100644 --- a/src/utils/ai/openaiShared.ts +++ b/src/utils/ai/openaiShared.ts @@ -3,6 +3,15 @@ import type { Tiktoken, TiktokenBPE } from "js-tiktoken/lite" import type { StatusCallback, AiAssistantAPIError } from "./aiAssistant" import { StreamingError, RefusalError, MaxTokensError } from "./shared" +export function isReasoningRejection(error: unknown): boolean { + if (!(error instanceof OpenAI.APIError) || error.status !== 400) return false + const param = (error as { param?: string | null }).param + return ( + (typeof param === "string" && param.includes("reasoning")) || + error.message.toLowerCase().includes("reasoning") + ) +} + let tiktokenEncoder: Tiktoken | null = null export async function countTokensFromNativePayload( diff --git a/src/utils/ai/registry.ts b/src/utils/ai/registry.ts index 947b42e37..85a9a21e6 100644 --- a/src/utils/ai/registry.ts +++ b/src/utils/ai/registry.ts @@ -10,6 +10,23 @@ type ProviderOptions = { baseURL?: string contextWindow?: number isCustom?: boolean + reasoning?: { effort: "high"; models: string[] } +} + +const reasoningOptions = ( + providerId: ProviderId, + settings?: AiAssistantSettings, +): Pick => { + const providerSettings = settings?.providers?.[providerId] + if ( + providerSettings?.reasoningEffort !== "high" || + !providerSettings.reasoningModels?.length + ) { + return {} + } + return { + reasoning: { effort: "high", models: providerSettings.reasoningModels }, + } } export function createProvider( @@ -20,7 +37,12 @@ export function createProvider( // Check built-in providers first const builtin = BUILTIN_PROVIDERS[providerId] if (builtin) { - return createProviderByType(builtin.type, providerId, apiKey) + return createProviderByType( + builtin.type, + providerId, + apiKey, + reasoningOptions(providerId, settings), + ) } // Check custom providers diff --git a/src/utils/ai/settings.test.ts b/src/utils/ai/settings.test.ts index b12d75713..e1565bc40 100644 --- a/src/utils/ai/settings.test.ts +++ b/src/utils/ai/settings.test.ts @@ -1,11 +1,13 @@ -import { describe, it, expect, afterEach, beforeAll } from "vitest" +import { describe, it, expect } from "vitest" import { reconcileSettings, getSelectedModel, getAiPermissions, - MODEL_OPTIONS, + getAllModelOptions, + getNextModel, + getUtilityModel, + providerForModel, } from "./settings" -import type { ModelOption } from "./settings" import type { AiAssistantSettings } from "../../providers/LocalStorageProvider/types" @@ -17,76 +19,119 @@ const makeSettings = ( }) describe("reconcileSettings", () => { - it("removes stale model IDs from enabledModels", () => { + it("keeps built-in model ids it does not recognize", () => { + // Given enabled models that no fixed list knows about const settings = makeSettings({ providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini", "removed-model", "also-removed"], + enabledModels: ["gpt-7", "gpt-5-mini"], grantSchemaAccess: false, }, }, }) + // When settings reconcile const result = reconcileSettings(settings) - expect(result.providers.openai!.enabledModels).toEqual(["gpt-5-mini"]) + // Then availability is the picker's job, not reconcile's + expect(result.providers.openai!.enabledModels).toEqual([ + "gpt-7", + "gpt-5-mini", + ]) }) - it("does not add defaultEnabled models when user has valid models", () => { + it("collapses legacy reasoning variants into plain ids", () => { const settings = makeSettings({ providers: { - anthropic: { + openai: { apiKey: "sk-test", - enabledModels: ["claude-sonnet-4-5"], + enabledModels: [ + "gpt-5.4@reasoning=high", + "gpt-5.4@reasoning=medium", + "gpt-5-mini", + ], grantSchemaAccess: false, }, }, }) const result = reconcileSettings(settings) - expect(result.providers.anthropic!.enabledModels).toEqual([ - "claude-sonnet-4-5", + expect(result.providers.openai!.enabledModels).toEqual([ + "gpt-5.4", + "gpt-5-mini", ]) }) - it("leaves enabledModels empty when all previous models were removed", () => { + it("folds a selected high variant into reasoningEffort", () => { + // Given a user who ran the high variant const settings = makeSettings({ + selectedModel: "gpt-5.4@reasoning=high", providers: { - anthropic: { + openai: { apiKey: "sk-test", - enabledModels: ["removed-model-1", "removed-model-2"], + enabledModels: ["gpt-5.4@reasoning=high", "gpt-5.4@reasoning=low"], grantSchemaAccess: false, }, }, }) + // When settings reconcile const result = reconcileSettings(settings) - expect(result.providers.anthropic!.enabledModels).toEqual([]) + // Then the provider runs on High and the selection is the plain id + expect(result.providers.openai!.reasoningEffort).toBe("high") + expect(result.selectedModel).toBe("gpt-5.4") }) - it("does not add defaults for unconfigured providers", () => { + it("migrates medium and low variant users to the provider default", () => { const settings = makeSettings({ + selectedModel: "gpt-5.4@reasoning=medium", providers: { - anthropic: { + openai: { apiKey: "sk-test", - enabledModels: ["claude-sonnet-4-5"], + enabledModels: ["gpt-5.4@reasoning=medium", "gpt-5.4@reasoning=low"], + grantSchemaAccess: false, + }, + }, + }) + const result = reconcileSettings(settings) + expect(result.providers.openai!.reasoningEffort).toBeUndefined() + expect(result.selectedModel).toBe("gpt-5.4") + }) + + it("removes custom models missing from their provider definition", () => { + const settings = makeSettings({ + customProviders: { + "custom-1": { + type: "openai-chat-completions", + name: "Test", + baseURL: "http://localhost:11434/v1", + contextWindow: 100_000, + models: ["llm-a"], + }, + }, + providers: { + "custom-1": { + apiKey: "", + enabledModels: ["custom-1:llm-a", "custom-1:llm-removed"], grantSchemaAccess: false, }, }, }) const result = reconcileSettings(settings) - expect(result.providers.openai).toBeUndefined() + expect(result.providers["custom-1"]!.enabledModels).toEqual([ + "custom-1:llm-a", + ]) }) it("is idempotent", () => { const settings = makeSettings({ - selectedModel: "claude-sonnet-4-5", + selectedModel: "gpt-5.4@reasoning=high", providers: { anthropic: { apiKey: "sk-test", - enabledModels: ["claude-sonnet-4-5", "stale-model"], + enabledModels: ["claude-sonnet-4-5"], grantSchemaAccess: true, }, openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini"], + enabledModels: ["gpt-5.4@reasoning=high", "gpt-5-mini"], grantSchemaAccess: false, }, }, @@ -117,7 +162,7 @@ describe("reconcileSettings", () => { ) }) - it("clears selectedModel if not in any enabledModels", () => { + it("repairs selectedModel when it is not enabled anywhere", () => { const settings = makeSettings({ selectedModel: "removed-model", providers: { @@ -158,7 +203,7 @@ describe("reconcileSettings", () => { providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini", "stale-model"], + enabledModels: ["gpt-5-mini", "gpt-5.4@reasoning=high"], grantSchemaAccess: false, }, }, @@ -184,7 +229,7 @@ describe("getSelectedModel", () => { expect(getSelectedModel(settings)).toBe("gpt-5-mini") }) - it("does not return selectedModel if not in enabledModels", () => { + it("falls back to the first enabled model", () => { const settings = makeSettings({ selectedModel: "claude-sonnet-4-5", providers: { @@ -195,7 +240,6 @@ describe("getSelectedModel", () => { }, }, }) - expect(getSelectedModel(settings)).not.toBe("claude-sonnet-4-5") expect(getSelectedModel(settings)).toBe("gpt-5-mini") }) @@ -205,180 +249,137 @@ describe("getSelectedModel", () => { }) }) -/** - * Simulates version upgrades by temporarily replacing MODEL_OPTIONS contents. - * Tests verify that user settings from a previous version are handled correctly - * when the app is updated with a different model list. - */ -describe("version compatibility scenarios", () => { - // Snapshot once before any mutation — re-snapshotting per call leaks an empty baseline on throw. - let originalOptions: ModelOption[] - beforeAll(() => { - originalOptions = [...MODEL_OPTIONS] - }) - - function setModelOptions(options: ModelOption[]) { - MODEL_OPTIONS.length = 0 - MODEL_OPTIONS.push(...options) - } - - afterEach(() => { - MODEL_OPTIONS.length = 0 - MODEL_OPTIONS.push(...originalOptions) - }) - - it("upgrade: model removed, selectedModel was that model", () => { - // v1: user had model-A and model-B, selected model-A - setModelOptions([ - { label: "A", value: "model-a", provider: "openai" }, - { label: "B", value: "model-b", provider: "openai" }, - ]) - - const v1Settings = makeSettings({ - selectedModel: "model-a", +describe("getAllModelOptions", () => { + it("builds options from enabled models with stored labels", () => { + // Given a provider with one stored label and one without + const settings = makeSettings({ providers: { openai: { apiKey: "sk-test", - enabledModels: ["model-a", "model-b"], + enabledModels: ["gpt-5.4", "gpt-5-mini"], grantSchemaAccess: false, + modelLabels: { "gpt-5.4": "GPT-5.4 (Custom)" }, }, }, }) + // When options build + const options = getAllModelOptions(settings) + // Then the stored label wins and the formatter fills the gap + expect(options).toEqual([ + { label: "GPT-5.4 (Custom)", value: "gpt-5.4", provider: "openai" }, + { label: "GPT-5 Mini", value: "gpt-5-mini", provider: "openai" }, + ]) + }) - // v2: model-A removed, model-C added - setModelOptions([ - { label: "B", value: "model-b", provider: "openai" }, - { - label: "C", - value: "model-c", - provider: "openai", - defaultEnabled: true, + it("includes namespaced custom provider models", () => { + const settings = makeSettings({ + customProviders: { + "custom-1": { + type: "openai-chat-completions", + name: "Test", + baseURL: "http://localhost:11434/v1", + contextWindow: 100_000, + models: ["llm-a"], + }, }, + }) + expect(getAllModelOptions(settings)).toEqual([ + { label: "llm-a", value: "custom-1:llm-a", provider: "custom-1" }, ]) - - const reconciled = reconcileSettings(v1Settings) - expect(reconciled.providers.openai!.enabledModels).toEqual(["model-b"]) - expect(reconciled.selectedModel).toBe("model-b") }) +}) - it("upgrade: all models removed for a provider", () => { - setModelOptions([{ label: "A", value: "model-a", provider: "openai" }]) - - const v1Settings = makeSettings({ - selectedModel: "model-a", +describe("providerForModel", () => { + it("finds the built-in provider that enabled the model", () => { + const settings = makeSettings({ providers: { - openai: { + anthropic: { apiKey: "sk-test", - enabledModels: ["model-a"], + enabledModels: ["claude-sonnet-5"], grantSchemaAccess: false, }, }, }) + expect(providerForModel("claude-sonnet-5", settings)).toBe("anthropic") + expect(providerForModel("gpt-5-mini", settings)).toBeNull() + }) - // v2: provider's models completely replaced - setModelOptions([ - { - label: "X", - value: "model-x", - provider: "openai", - defaultEnabled: true, - }, - { label: "Y", value: "model-y", provider: "openai" }, - ]) - - const reconciled = reconcileSettings(v1Settings) - // all old models gone, empty list — user must re-enable in settings - expect(reconciled.providers.openai!.enabledModels).toEqual([]) - expect(reconciled.selectedModel).toBeUndefined() - expect(getSelectedModel(reconciled)).toBeNull() + it("parses namespaced custom values without settings lookup", () => { + expect(providerForModel("custom-1:llm-a")).toBe("custom-1") }) +}) - it("upgrade: new models added, user keeps their selection", () => { - setModelOptions([ - { label: "A", value: "model-a", provider: "anthropic", default: true }, - { label: "B", value: "model-b", provider: "anthropic" }, - ]) +describe("getNextModel", () => { + it("keeps the current model while it stays enabled", () => { + expect( + getNextModel("gpt-5-mini", { openai: ["gpt-5.4", "gpt-5-mini"] }), + ).toBe("gpt-5-mini") + }) - const v1Settings = makeSettings({ - selectedModel: "model-b", + it("takes the first enabled model of any provider when the current one is gone", () => { + const settings = makeSettings({ providers: { anthropic: { apiKey: "sk-test", - enabledModels: ["model-a", "model-b"], - grantSchemaAccess: true, + enabledModels: ["claude-sonnet-5"], + grantSchemaAccess: false, }, }, }) - - // v2: model-C added - setModelOptions([ - { label: "A", value: "model-a", provider: "anthropic", default: true }, - { label: "B", value: "model-b", provider: "anthropic" }, - { - label: "C", - value: "model-c", - provider: "anthropic", - defaultEnabled: true, - }, - ]) - - const reconciled = reconcileSettings(v1Settings) - // existing models preserved, new model NOT auto-added - expect(reconciled.providers.anthropic!.enabledModels).toEqual([ - "model-a", - "model-b", - ]) - expect(reconciled.selectedModel).toBe("model-b") - expect(getSelectedModel(reconciled)).toBe("model-b") + expect( + getNextModel("gpt-5-mini", { anthropic: ["claude-sonnet-5"] }, settings), + ).toBe("claude-sonnet-5") }) - it("upgrade: selected model survives but some enabled models removed", () => { - setModelOptions([ - { label: "A", value: "model-a", provider: "openai" }, - { label: "B", value: "model-b", provider: "openai" }, - { label: "C", value: "model-c", provider: "openai" }, - ]) + it("returns null when nothing is enabled", () => { + expect(getNextModel("gpt-5-mini", {})).toBeNull() + }) +}) - const v1Settings = makeSettings({ - selectedModel: "model-b", +describe("getUtilityModel", () => { + it("returns the persisted utility model for a built-in provider", () => { + const settings = makeSettings({ + selectedModel: "gpt-5.4", providers: { openai: { apiKey: "sk-test", - enabledModels: ["model-a", "model-b", "model-c"], + enabledModels: ["gpt-5.4"], grantSchemaAccess: false, + utilityModel: "gpt-5.6-luna", }, }, }) - - // v2: model-A and model-C removed - setModelOptions([ - { label: "B", value: "model-b", provider: "openai" }, - { label: "D", value: "model-d", provider: "openai" }, - ]) - - const reconciled = reconcileSettings(v1Settings) - expect(reconciled.providers.openai!.enabledModels).toEqual(["model-b"]) - expect(reconciled.selectedModel).toBe("model-b") - expect(getSelectedModel(reconciled)).toBe("model-b") + expect(getUtilityModel("openai", settings)).toBe("gpt-5.6-luna") }) - it("downgrade: user has models from a newer version", () => { - setModelOptions([{ label: "A", value: "model-a", provider: "openai" }]) - - const futureSettings = makeSettings({ - selectedModel: "model-future", + it("falls back to the selected model when nothing is persisted", () => { + const settings = makeSettings({ + selectedModel: "gpt-5.4", providers: { openai: { apiKey: "sk-test", - enabledModels: ["model-a", "model-future"], + enabledModels: ["gpt-5.4"], grantSchemaAccess: false, }, }, }) + expect(getUtilityModel("openai", settings)).toBe("gpt-5.4") + }) - const reconciled = reconcileSettings(futureSettings) - expect(reconciled.providers.openai!.enabledModels).toEqual(["model-a"]) - expect(reconciled.selectedModel).toBe("model-a") + it("uses the selected model for custom providers", () => { + const settings = makeSettings({ + selectedModel: "custom-1:llm-a", + customProviders: { + "custom-1": { + type: "openai-chat-completions", + name: "Test", + baseURL: "http://localhost:11434/v1", + contextWindow: 100_000, + models: ["llm-a"], + }, + }, + }) + expect(getUtilityModel("custom-1", settings)).toBe("custom-1:llm-a") }) }) diff --git a/src/utils/ai/settings.ts b/src/utils/ai/settings.ts index 65bdaf8bf..19bff7bcb 100644 --- a/src/utils/ai/settings.ts +++ b/src/utils/ai/settings.ts @@ -5,6 +5,15 @@ import type { import type { Permissions } from "../tools/permissions" import { getValue } from "../localStorage" import { StoreKey } from "../localStorage/types" +import { + computeReasoningModels, + filterOpenAiChatModels, + formatModelLabel, + matchesListedModel, + resolveUtilityModel, + UTILITY_MODEL_TIERS, +} from "./modelCatalog" +import type { ProviderModel } from "./modelCatalog" export type ProviderType = "anthropic" | "openai" | "openai-chat-completions" @@ -38,76 +47,6 @@ export type ModelOption = { label: string value: string provider: ProviderId - isSlow?: boolean - isTestModel?: boolean - default?: boolean - defaultEnabled?: boolean -} - -export const MODEL_OPTIONS: ModelOption[] = [ - { - label: "Claude Opus 4.7", - value: "claude-opus-4-7", - provider: "anthropic", - isSlow: true, - defaultEnabled: true, - default: true, - }, - { - label: "Claude Sonnet 4.6", - value: "claude-sonnet-4-6", - provider: "anthropic", - defaultEnabled: true, - }, - { - label: "Claude Sonnet 4.5", - value: "claude-sonnet-4-5", - provider: "anthropic", - }, - { - label: "Claude Haiku 4.5", - value: "claude-haiku-4-5", - provider: "anthropic", - isTestModel: true, - }, - { - label: "GPT-5.4 (High Reasoning)", - value: "gpt-5.4@reasoning=high", - provider: "openai", - }, - { - label: "GPT-5.4 (Medium Reasoning)", - value: "gpt-5.4@reasoning=medium", - provider: "openai", - defaultEnabled: true, - default: true, - }, - { - label: "GPT-5.4 (Low Reasoning)", - value: "gpt-5.4@reasoning=low", - provider: "openai", - defaultEnabled: true, - }, - { - label: "GPT-5 mini", - value: "gpt-5-mini", - provider: "openai", - defaultEnabled: true, - }, - { - label: "GPT-5 nano", - value: "gpt-5-nano", - provider: "openai", - defaultEnabled: true, - isTestModel: true, - }, -] - -export type ReasoningEffort = "high" | "medium" | "low" - -export type ModelProps = { - model: string - reasoningEffort?: ReasoningEffort } const CUSTOM_MODEL_SEP = ":" @@ -131,67 +70,63 @@ export const parseModelValue = ( } } +export const getModelLabel = ( + modelId: string, + providerId: ProviderId, + settings?: AiAssistantSettings, +): string => + settings?.providers?.[providerId]?.modelLabels?.[modelId] ?? + formatModelLabel(modelId) + export const getAllModelOptions = ( settings?: AiAssistantSettings, ): ModelOption[] => { - if (!settings?.customProviders) return MODEL_OPTIONS - const customModels: ModelOption[] = [] - for (const [providerId, def] of Object.entries(settings.customProviders)) { + if (!settings) return [] + const options: ModelOption[] = [] + for (const providerId of Object.keys(BUILTIN_PROVIDERS)) { + const enabledModels = settings.providers?.[providerId]?.enabledModels ?? [] + for (const modelId of enabledModels) { + options.push({ + label: getModelLabel(modelId, providerId, settings), + value: modelId, + provider: providerId, + }) + } + } + for (const [providerId, def] of Object.entries( + settings.customProviders ?? {}, + )) { for (const modelId of def.models) { - customModels.push({ + options.push({ label: modelId, value: makeCustomModelValue(providerId, modelId), provider: providerId, }) } } - return [...MODEL_OPTIONS, ...customModels] + return options } export const providerForModel = ( model: ModelOption["value"], - _settings?: AiAssistantSettings, + settings?: AiAssistantSettings, ): ProviderId | null => { // Check for namespaced custom model value (providerId:modelId) const parsed = parseModelValue(model) if ("customProviderId" in parsed) return parsed.customProviderId - // Fall back to built-in model lookup - return MODEL_OPTIONS.find((m) => m.value === model)?.provider ?? null -} - -export const getModelProps = (model: ModelOption["value"]): ModelProps => { - const { rawModel } = parseModelValue(model) - const parts = rawModel.split("@") - const modelName = parts[0] - const extraParams = parts[1] - ?.split(",") - ?.map((p) => ({ key: p.split("=")[0], value: p.split("=")[1] })) - if (extraParams) { - const reasoningParam = extraParams.find((p) => p.key === "reasoning") - if (reasoningParam && reasoningParam.value) { - return { - model: modelName, - reasoningEffort: reasoningParam.value as ReasoningEffort, - } - } - } - return { model: modelName } + return ( + Object.keys(BUILTIN_PROVIDERS).find((providerId) => + settings?.providers?.[providerId]?.enabledModels?.includes(model), + ) ?? null + ) } export const getAllProviders = ( settings?: AiAssistantSettings, -): ProviderId[] => { - const providers = new Set() - MODEL_OPTIONS.forEach((model) => { - providers.add(model.provider) - }) - if (settings?.customProviders) { - for (const id of Object.keys(settings.customProviders)) { - providers.add(id) - } - } - return Array.from(providers) -} +): ProviderId[] => [ + ...Object.keys(BUILTIN_PROVIDERS), + ...Object.keys(settings?.customProviders ?? {}), +] export const getSelectedModel = ( settings: AiAssistantSettings, @@ -205,16 +140,7 @@ export const getSelectedModel = ( ) { return selectedModel } - - const allModels = getAllModelOptions(settings) - // Fall back to first enabled default model, then first enabled model - return ( - enabledModels.find( - (id) => allModels.find((m) => m.value === id)?.default, - ) ?? - enabledModels[0] ?? - null - ) + return enabledModels[0] ?? null } export const getAllEnabledModels = ( @@ -241,37 +167,28 @@ export const getNextModel = ( enabledModels: Record, settings?: AiAssistantSettings, ): string | null => { - let nextModel: string | null | undefined = currentModel - - const allModels = getAllModelOptions(settings) - const modelProvider = currentModel - ? providerForModel(currentModel, settings) - : null + const providerOf = (model: string) => { + const parsed = parseModelValue(model) + if ("customProviderId" in parsed) return parsed.customProviderId + return ( + Object.keys(enabledModels).find((p) => + enabledModels[p]?.includes(model), + ) ?? providerForModel(model, settings) + ) + } + const modelProvider = currentModel ? providerOf(currentModel) : null if (modelProvider && enabledModels[modelProvider]?.length > 0) { - // Current model is still enabled, so we can use it if (currentModel && enabledModels[modelProvider].includes(currentModel)) { return currentModel } - // Take the default model of this provider, otherwise the first enabled model of this provider - nextModel = - enabledModels[modelProvider].find( - (m) => allModels.find((mo) => mo.value === m)?.default, - ) ?? enabledModels[modelProvider][0] - } else { - // No other enabled models for this provider, we have to choose from another provider if exists - const otherProviderWithEnabledModel = getAllProviders(settings).find( - (p) => enabledModels[p]?.length > 0, - ) - if (otherProviderWithEnabledModel) { - nextModel = - enabledModels[otherProviderWithEnabledModel].find( - (m) => allModels.find((mo) => mo.value === m)?.default, - ) ?? enabledModels[otherProviderWithEnabledModel][0] - } else { - nextModel = null - } + return enabledModels[modelProvider][0] } - return nextModel ?? null + const providerWithEnabledModel = getAllProviders(settings).find( + (p) => enabledModels[p]?.length > 0, + ) + return providerWithEnabledModel + ? enabledModels[providerWithEnabledModel][0] + : null } export const isAiAssistantConfigured = ( @@ -288,19 +205,52 @@ export const canUseAiAssistant = (settings: AiAssistantSettings): boolean => { return isAiAssistantConfigured(settings) && !!settings.selectedModel } -export const getTestModel = ( +export const getUtilityModel = ( providerId: ProviderId, settings?: AiAssistantSettings, ): string | null => { - if (settings?.customProviders?.[providerId]) { + if (!settings) return null + if (settings.customProviders?.[providerId]) { return settings.selectedModel ?? null } return ( - MODEL_OPTIONS.find((m) => m.provider === providerId && m.isTestModel) - ?.value ?? null + settings.providers?.[providerId]?.utilityModel ?? getSelectedModel(settings) ) } +/** + * Derives the listing-dependent provider settings captured at Save time: + * labels for enabled models, the utility model, and the reasoning gate. + */ +export type ListingMetadata = { + modelLabels: Record + utilityModel?: string + reasoningModels?: string[] +} + +export const buildListingMetadata = ( + providerId: ProviderId, + listing: ProviderModel[], + enabledModels: string[], +): ListingMetadata => { + const isOpenAi = BUILTIN_PROVIDERS[providerId]?.type === "openai" + const utilityPool = isOpenAi ? filterOpenAiChatModels(listing) : listing + const utilityModel = resolveUtilityModel( + utilityPool, + UTILITY_MODEL_TIERS[isOpenAi ? "openai" : "anthropic"], + ) + const modelLabels: Record = {} + for (const id of enabledModels) { + const label = listing.find((m) => matchesListedModel(id, m.id))?.label + if (label) modelLabels[id] = label + } + return { + modelLabels, + ...(utilityModel ? { utilityModel } : {}), + ...(isOpenAi ? { reasoningModels: computeReasoningModels(listing) } : {}), + } +} + /** * Returns the context window for a given provider. * For custom providers, returns the configured value. @@ -314,10 +264,17 @@ export const getProviderContextWindow = ( return custom?.contextWindow ?? null } +const LEGACY_REASONING_VARIANT = /@reasoning=(high|medium|low)$/ + +const collapseLegacyVariant = (modelId: string): string => + modelId.replace(LEGACY_REASONING_VARIANT, "") + /** - * Reconciles persisted AI assistant settings against current model options. - * Removes stale model IDs from built-in providers' enabledModels. - * Preserves custom provider models (validated against customProviders definitions). + * Reconciles persisted AI assistant settings. + * Collapses legacy `@reasoning=` model variants into plain ids and folds a + * selected high variant into the provider-level reasoningEffort. + * Validates custom provider models against customProviders definitions; + * built-in models stay until the Manage Models picker removes them. * * Pure function — does not write to localStorage. * Idempotent: applying it multiple times produces the same result. @@ -325,25 +282,42 @@ export const getProviderContextWindow = ( export const reconcileSettings = ( settings: AiAssistantSettings, ): AiAssistantSettings => { - const allValidIds = new Set(getAllModelOptions(settings).map((m) => m.value)) const result = { ...settings, providers: { ...settings.providers }, } + const selectedModel = result.selectedModel for (const providerKey of Object.keys(result.providers)) { const providerSettings = result.providers[providerKey] if (!providerSettings?.enabledModels) continue - const models = providerSettings.enabledModels.filter((id) => - allValidIds.has(id), - ) + const selectedHighVariant = + selectedModel !== undefined && + selectedModel.endsWith("@reasoning=high") && + providerSettings.enabledModels.includes(selectedModel) + const collapsed = [ + ...new Set(providerSettings.enabledModels.map(collapseLegacyVariant)), + ] + const validCustomIds = settings.customProviders?.[providerKey] + ? new Set( + settings.customProviders[providerKey].models.map((m) => + makeCustomModelValue(providerKey, m), + ), + ) + : null result.providers[providerKey] = { ...providerSettings, - enabledModels: models, + enabledModels: BUILTIN_PROVIDERS[providerKey] + ? collapsed + : collapsed.filter((id) => validCustomIds?.has(id)), + ...(selectedHighVariant ? { reasoningEffort: "high" as const } : {}), } } + if (result.selectedModel !== undefined) { + result.selectedModel = collapseLegacyVariant(result.selectedModel) + } result.selectedModel = getSelectedModel(result) ?? undefined return result diff --git a/src/utils/ai/types.ts b/src/utils/ai/types.ts index e0950b921..4f1a7e5f8 100644 --- a/src/utils/ai/types.ts +++ b/src/utils/ai/types.ts @@ -8,8 +8,11 @@ import type { import type { Permissions, ToolCategory } from "../tools/permissions" import type { ValidateQueryResult } from "../questdb/types" import type { ProviderId } from "./settings" +import type { ProviderModel } from "./modelCatalog" import type { ToolExecutionContext } from "./shared" +export { type ProviderModel } + export type ToolSurface = "ai" | "mcp" export interface ToolDefinition { @@ -97,18 +100,13 @@ export interface AIProvider { abortSignal?: AbortSignal }): Promise - testConnection(params: { - apiKey: string - model: string - }): Promise<{ valid: boolean; error?: string }> - countTokens(params: { messages: Message[] systemPrompt: string model: string }): Promise - listModels(): Promise + listModels(): Promise classifyError(error: unknown, setStatus: StatusCallback): AiAssistantAPIError isNonRetryableError(error: unknown): boolean From 781c40b91881af71a4b317bd47aca9b474834cd8 Mon Sep 17 00:00:00 2001 From: emrberk Date: Thu, 3 Sep 2026 19:37:14 +0300 Subject: [PATCH 2/9] feat: hide pre-gpt-5 generations from the default model picker The picker's default view now shares the gpt-5-launch date gate with the reasoning control: older generations (gpt-3.5/4/4o/o-series) move behind the "Show all models" escape hatch and stay manually addable. 16 rows against today's live listing instead of 30. Co-Authored-By: Claude Fable 5 --- src/utils/ai/modelCatalog.test.ts | 43 +++++++++++++++++++++---------- src/utils/ai/modelCatalog.ts | 17 +++++++----- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/utils/ai/modelCatalog.test.ts b/src/utils/ai/modelCatalog.test.ts index ed59a8821..d700a289b 100644 --- a/src/utils/ai/modelCatalog.test.ts +++ b/src/utils/ai/modelCatalog.test.ts @@ -74,15 +74,15 @@ describe("filterOpenAiChatModels", () => { it("drops known non-chat models and dated snapshots", () => { // Given a listing with chat models, noise, and dated snapshots const listing = [ - model("gpt-5.4", 300), - model("gpt-5.4-2026-03-05", 300), - model("text-embedding-3-small", 1), - model("whisper-1", 1), - model("gpt-4o-mini-tts", 1), - model("gpt-5-chat-latest", 200), - model("gpt-5.3-codex", 250), - model("sora-2", 250), - model("davinci-002", 1), + model("gpt-5.4", AUG_2025 + 300), + model("gpt-5.4-2026-03-05", AUG_2025 + 300), + model("text-embedding-3-small", AUG_2025 + 1), + model("whisper-1", AUG_2025 + 1), + model("gpt-4o-mini-tts", AUG_2025 + 1), + model("gpt-5-chat-latest", AUG_2025 + 200), + model("gpt-5.3-codex", AUG_2025 + 250), + model("sora-2", AUG_2025 + 250), + model("davinci-002", AUG_2025 + 1), ] // When the filter runs const kept = filterOpenAiChatModels(listing).map((m) => m.id) @@ -90,18 +90,33 @@ describe("filterOpenAiChatModels", () => { expect(kept).toEqual(["gpt-5.4"]) }) + it("hides generations older than gpt-5 by default", () => { + // Given chat models from before and after the gpt-5 launch + const listing = [ + model("gpt-5", AUG_2025), + model("gpt-4.1", JUL_2025), + model("gpt-3.5-turbo", JUL_2025 - 1_000_000), + ] + // When the filter runs + const kept = filterOpenAiChatModels(listing).map((m) => m.id) + // Then only the current generation stays in the default view + expect(kept).toEqual(["gpt-5"]) + }) + it("keeps a brand-new generation without any code change", () => { - const kept = filterOpenAiChatModels([model("gpt-6", 500)]).map((m) => m.id) + const kept = filterOpenAiChatModels([model("gpt-6", AUG_2025 + 500)]).map( + (m) => m.id, + ) expect(kept).toEqual(["gpt-6"]) }) it("sorts newest first", () => { const kept = filterOpenAiChatModels([ - model("gpt-4.1", 100), - model("gpt-5.4", 300), - model("gpt-5", 200), + model("gpt-5", AUG_2025 + 100), + model("gpt-5.4", AUG_2025 + 300), + model("gpt-5.2", AUG_2025 + 200), ]).map((m) => m.id) - expect(kept).toEqual(["gpt-5.4", "gpt-5", "gpt-4.1"]) + expect(kept).toEqual(["gpt-5.4", "gpt-5.2", "gpt-5"]) }) }) diff --git a/src/utils/ai/modelCatalog.ts b/src/utils/ai/modelCatalog.ts index 86e4cc113..e6a3a5f68 100644 --- a/src/utils/ai/modelCatalog.ts +++ b/src/utils/ai/modelCatalog.ts @@ -6,6 +6,10 @@ export type ProviderModel = { const DATE_SUFFIX = /-(\d{8}|\d{4}-\d{2}-\d{2}|\d{4})$/ +// Everything since the gpt-5 launch reasons, and belongs in the default +// picker view; older generations stay reachable via "Show all models". +const GPT5_LAUNCH_START = Date.UTC(2025, 7, 1) / 1000 + const OPENAI_NON_CHAT_TOKENS = [ "embedding", "tts", @@ -76,7 +80,12 @@ const isOpenAiNonChatModel = (id: string): boolean => export const filterOpenAiChatModels = ( models: ProviderModel[], ): ProviderModel[] => - sortModelsNewestFirst(models.filter((m) => !isOpenAiNonChatModel(m.id))) + sortModelsNewestFirst( + models.filter( + (m) => + !isOpenAiNonChatModel(m.id) && (m.created ?? 0) >= GPT5_LAUNCH_START, + ), + ) export const UTILITY_MODEL_TIERS: Record<"anthropic" | "openai", string[]> = { anthropic: ["haiku", "sonnet"], @@ -104,12 +113,8 @@ export const resolveUtilityModel = ( return null } -const REASONING_GATE_START = Date.UTC(2025, 7, 1) / 1000 - export const computeReasoningModels = (models: ProviderModel[]): string[] => - models - .filter((m) => (m.created ?? 0) >= REASONING_GATE_START) - .map((m) => m.id) + models.filter((m) => (m.created ?? 0) >= GPT5_LAUNCH_START).map((m) => m.id) export const isReasoningModel = ( modelId: string, From 978d318059df028cfe4b27be52ec5db475fedc33 Mon Sep 17 00:00:00 2001 From: emrberk Date: Thu, 3 Sep 2026 19:43:56 +0300 Subject: [PATCH 3/9] feat: drop research-tier pro models from the default picker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gpt-5-pro and friends reason for minutes and stream no output or reasoning summaries even with summary auto (verified live) — in an interactive chat they read as unresponsive. They stay reachable via Show all models and manual add. 12 default rows against today's listing. Co-Authored-By: Claude Fable 5 --- src/utils/ai/modelCatalog.test.ts | 1 + src/utils/ai/modelCatalog.ts | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/utils/ai/modelCatalog.test.ts b/src/utils/ai/modelCatalog.test.ts index d700a289b..8ff1f11b4 100644 --- a/src/utils/ai/modelCatalog.test.ts +++ b/src/utils/ai/modelCatalog.test.ts @@ -81,6 +81,7 @@ describe("filterOpenAiChatModels", () => { model("gpt-4o-mini-tts", AUG_2025 + 1), model("gpt-5-chat-latest", AUG_2025 + 200), model("gpt-5.3-codex", AUG_2025 + 250), + model("gpt-5.4-pro", AUG_2025 + 250), model("sora-2", AUG_2025 + 250), model("davinci-002", AUG_2025 + 1), ] diff --git a/src/utils/ai/modelCatalog.ts b/src/utils/ai/modelCatalog.ts index e6a3a5f68..c872e2b8f 100644 --- a/src/utils/ai/modelCatalog.ts +++ b/src/utils/ai/modelCatalog.ts @@ -26,6 +26,9 @@ const OPENAI_NON_CHAT_TOKENS = [ "codex", "computer-use", "chat-latest", + // Research-tier models: minutes of reasoning with no streamed output or + // summaries — they read as unresponsive in an interactive chat. + "-pro", "chatgpt", "babbage", "davinci", From 4265ae382eeedee8e03a8fc4e47893da8ca404ab Mon Sep 17 00:00:00 2001 From: emrberk Date: Fri, 4 Sep 2026 14:58:48 +0300 Subject: [PATCH 4/9] reviews --- e2e/commands.js | 3 +- e2e/tests/console/aiAssistant.spec.js | 539 +--------- e2e/tests/console/aiProviderSetup.spec.js | 993 ++++++++++++++++++ e2e/utils/aiAssistant.js | 35 + src/components/MultiStepModal/index.tsx | 33 +- .../SetupAIAssistant/ConfigurationModal.tsx | 41 +- .../SetupAIAssistant/ManageModelsModal.tsx | 240 +++-- .../SetupAIAssistant/ModelPicker.tsx | 63 +- .../SetupAIAssistant/ReasoningSection.tsx | 18 +- .../SetupAIAssistant/SettingsModal.tsx | 222 +++- src/components/ValidationNotice/index.tsx | 37 + src/providers/LocalStorageProvider/index.tsx | 18 + src/providers/LocalStorageProvider/types.ts | 1 - src/utils/ai/anthropicProvider.ts | 14 +- src/utils/ai/index.ts | 5 +- src/utils/ai/modelCatalog.test.ts | 54 +- src/utils/ai/modelCatalog.ts | 17 +- src/utils/ai/openaiChatCompletionsProvider.ts | 67 +- src/utils/ai/openaiProvider.fallback.test.ts | 164 +++ src/utils/ai/openaiProvider.ts | 85 +- src/utils/ai/openaiShared.test.ts | 50 + src/utils/ai/openaiShared.ts | 6 +- src/utils/ai/reasoningFallback.ts | 16 + src/utils/ai/registry.ts | 16 +- src/utils/ai/settings.test.ts | 122 ++- src/utils/ai/settings.ts | 64 +- 26 files changed, 1917 insertions(+), 1006 deletions(-) create mode 100644 e2e/tests/console/aiProviderSetup.spec.js create mode 100644 src/components/ValidationNotice/index.tsx create mode 100644 src/utils/ai/openaiProvider.fallback.test.ts create mode 100644 src/utils/ai/openaiShared.test.ts create mode 100644 src/utils/ai/reasoningFallback.ts diff --git a/e2e/commands.js b/e2e/commands.js index 34f3e1e3d..905bbf2bf 100644 --- a/e2e/commands.js +++ b/e2e/commands.js @@ -4,7 +4,8 @@ require("@4tw/cypress-drag-drop") const { ctrlOrCmd, escapeRegExp, seedNotebookOnboarding } = require("./utils") -const baseUrl = Cypress.config("baseUrl") +const contextPath = process.env.QDB_HTTP_CONTEXT_WEB_CONSOLE || "" +const baseUrl = `http://localhost:9999${contextPath}` const tableSchemas = { btc_trades: diff --git a/e2e/tests/console/aiAssistant.spec.js b/e2e/tests/console/aiAssistant.spec.js index 06d98d7aa..39e967ecf 100644 --- a/e2e/tests/console/aiAssistant.spec.js +++ b/e2e/tests/console/aiAssistant.spec.js @@ -2,6 +2,7 @@ const { PROVIDERS, + interceptAIChatRequest, CUSTOM_PROVIDER_DEFAULTS, getOpenAIConfiguredSettings, getAnthropicConfiguredSettings, @@ -46,131 +47,6 @@ function interceptAIRequestWithResponse( }).as(aliasName) } -/** - * Intercepts AI chat requests with a default test response. - * - * @param {"anthropic" | "openai"} provider - The AI provider to intercept - * @param {string} [alias] - Optional custom alias for the intercept - * @param {number} [delay=0] - Delay in milliseconds - * @param {Object} [options] - Options - * @param {boolean} [options.streaming=true] - Whether to use streaming response - */ -function interceptAIChatRequest( - provider, - alias, - delay = 200, - options = { streaming: true }, -) { - const aliasName = alias || `${provider}ChatRequest` - const endpoint = PROVIDERS[provider].endpoint - const { streaming = true } = options - - const responseData = createFinalResponseData( - provider, - "Test response explanation", - ) - - cy.intercept("POST", endpoint, (req) => { - if (isTitleRequest(provider, req.body)) { - req.reply(createChatTitleResponse(provider, "Test Chat")) - return - } - req.alias = aliasName - req.reply(createResponse(provider, responseData, { streaming, delay })) - }) -} - -/** - * Intercepts AI provider model listing requests. - * Validation now runs through GET /v1/models, so a mocked listing both - * validates the key and feeds the model picker. - * - * The OpenAI listing carries noise (whisper-1) that the picker must filter, - * and `created` timestamps that drive newest-first ordering. - * - * @param {"anthropic" | "openai"} provider - The AI provider to intercept - * @param {boolean} success - If true, returns 200 with a listing; if false, returns 401 - */ -function interceptTokenValidation(provider, success) { - if (provider === "openai") { - if (success) { - cy.intercept("GET", "https://api.openai.com/v1/models*", { - statusCode: 200, - delay: 200, - body: { - object: "list", - data: [ - { id: "gpt-5.4", object: "model", created: 1772000000 }, - { id: "gpt-5-mini", object: "model", created: 1754500000 }, - { id: "gpt-5", object: "model", created: 1754400000 }, - { id: "gpt-5-nano", object: "model", created: 1754300000 }, - { id: "whisper-1", object: "model", created: 1677532384 }, - ], - }, - }).as("openaiValidation") - } else { - cy.intercept("GET", "https://api.openai.com/v1/models*", { - statusCode: 401, - delay: 200, - body: { - error: { - message: - "Incorrect API key provided: ***. You can find your API key at https://platform.openai.com/account/api-keys.", - type: "invalid_request_error", - param: null, - code: "invalid_api_key", - }, - }, - }).as("openaiValidation") - } - } else if (provider === "anthropic") { - if (success) { - cy.intercept("GET", "https://api.anthropic.com/v1/models*", { - statusCode: 200, - delay: 200, - body: { - data: [ - { - type: "model", - id: "claude-opus-4-5", - display_name: "Claude Opus 4.5", - created_at: "2025-11-01T00:00:00Z", - }, - { - type: "model", - id: "claude-sonnet-4-5", - display_name: "Claude Sonnet 4.5", - created_at: "2025-09-29T00:00:00Z", - }, - { - type: "model", - id: "claude-haiku-4-5", - display_name: "Claude Haiku 4.5", - created_at: "2025-10-01T00:00:00Z", - }, - ], - has_more: false, - first_id: "claude-opus-4-5", - last_id: "claude-haiku-4-5", - }, - }).as("anthropicValidation") - } else { - cy.intercept("GET", "https://api.anthropic.com/v1/models*", { - statusCode: 401, - delay: 200, - body: { - type: "error", - error: { - type: "authentication_error", - message: "invalid x-api-key", - }, - request_id: "req_mock_test", - }, - }).as("anthropicValidation") - } - } -} - describe("ai assistant", () => { beforeEach(() => { cy.intercept("POST", PROVIDERS.openai.endpoint, (req) => { @@ -194,419 +70,6 @@ describe("ai assistant", () => { }).as("unhandledAnthropicModels") }) - describe("onboarding and settings", () => { - beforeEach(() => { - cy.loadConsoleWithAuth() - }) - - it("should display ai assistant promo", () => { - // When - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - - // Then - cy.getByDataHook("ai-promo-modal").should("be.visible") - - // When - cy.getByDataHook("ai-promo-close").should("be.visible").click() - - // Then - cy.getByDataHook("ai-promo-modal").should("not.exist") - - // When - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - cy.getByDataHook("ai-promo-continue").should("be.visible").click() - - // Then - cy.getByDataHook("ai-settings-modal-step-one").should("be.visible") - }) - - it("should handle invalid api key", () => { - // When - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - cy.getByDataHook("ai-promo-continue").should("be.visible").click() - - // Then - cy.getByDataHook("ai-settings-modal-step-one").should("be.visible") - // API key input is hidden until a provider is selected - cy.getByDataHook("ai-settings-api-key").should("not.exist") - - // When - select Anthropic - cy.getByDataHook("ai-settings-provider-anthropic").click() - - // Then - API key input appears - cy.getByDataHook("ai-settings-api-key") - .should("be.visible") - .should("have.attr", "placeholder", "Enter Anthropic API key") - - // When - switch to OpenAI - cy.getByDataHook("ai-settings-provider-openai").click() - - // Then - cy.getByDataHook("ai-settings-api-key") - .should("be.visible") - .should("have.attr", "placeholder", "Enter OpenAI API key") - ;["anthropic", "openai"].forEach((provider) => { - // Given - interceptTokenValidation(provider, false) - - // When - cy.getByDataHook(`ai-settings-provider-${provider}`).click() - - // Then - cy.getByDataHook("ai-settings-api-key") - .should("be.visible") - .should( - "have.attr", - "placeholder", - `Enter ${provider === "anthropic" ? "Anthropic" : "OpenAI"} API key`, - ) - .should("be.empty") - - // When - cy.getByDataHook("ai-settings-api-key").type("invalid-api-key") - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - cy.getByDataHook("multi-step-modal-next-button") - .should("be.disabled") - .should("contain", "Validating...") - - // When - cy.wait(`@${provider}Validation`) - - // Then - cy.getByDataHook("ai-settings-api-key-error").should("be.visible") - }) - }) - - it("should handle valid api key", () => { - // Given - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - cy.getByDataHook("ai-promo-continue").should("be.visible").click() - ;["anthropic", "openai"].forEach((provider) => { - // Given - interceptTokenValidation(provider, true) - - // When - cy.getByDataHook(`ai-settings-provider-${provider}`).click() - - // When - cy.getByDataHook("ai-settings-api-key").type("valid-api-key") - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") - cy.getByDataHook("multi-step-modal-cancel-button").click() - }) - }) - - it("should show ai buttons after setup is completed", () => { - // Given - interceptTokenValidation("openai", true) - - // When - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - cy.getByDataHook("ai-promo-continue").should("be.visible").click() - cy.getByDataHook("ai-settings-provider-openai").click() - cy.getByDataHook("ai-settings-api-key").type("valid-api-key") - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - step two shows the filtered listing, nothing preselected - cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") - cy.getByDataHook("configure-models-model-row").should("have.length", 4) - - // When - enable two models and activate - cy.getByDataHook("configure-models-model-row").contains("gpt-5.4").click() - cy.getByDataHook("configure-models-model-row") - .contains("gpt-5-mini") - .click() - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - cy.getByDataHook("ai-assistant-settings-button").should( - "contain", - "AI Settings", - ) - cy.getByDataHook("ai-chat-button").should("be.visible") - cy.getByDataHook("ai-settings-model-dropdown").should("be.visible") - - // When / Then — selecting a model closes the dropdown (handleModelSelect - // calls setDropdownActive(false)), so re-open it for each model and - // re-query the item just before clicking; otherwise the list detaches the - // node as it settles/closes and cy.click() hits a stale element. - ;[0, 1].forEach((index) => { - cy.getByDataHook("ai-settings-model-dropdown").click() - cy.getByDataHook("ai-settings-model-item").should("be.visible") - - cy.getByDataHook("ai-settings-model-item") - .eq(index) - .find("[data-hook='ai-settings-model-item-label']") - .invoke("text") - .then((text) => { - const label = text.trim() - cy.getByDataHook("ai-settings-model-item").eq(index).click() - cy.getByDataHook("ai-settings-model-dropdown").should( - "contain", - label, - ) - }) - }) - - // When - cy.typeQuery("SELECT 1;") - - // Then - cy.getAIIconInLine(1).should("be.visible") - - // When - cy.getByDataHook("ai-assistant-settings-button").click() - - // Then - cy.getByDataHook("ai-settings-validated-badge") - .should("be.visible") - .should("contain", "Validated") - cy.getByDataHook("ai-settings-provider-openai") - .getByDataHook("ai-settings-provider-status") - .should("be.visible") - .should("contain", "Enabled") - - cy.getByDataHook("ai-settings-provider-anthropic") - .getByDataHook("ai-settings-provider-status") - .should("be.visible") - .should("contain", "Inactive") - - // When - cy.getByDataHook("ai-settings-remove-provider").scrollIntoView() - cy.getByDataHook("ai-settings-remove-provider") - .should("be.visible") - .click() - - // Then - cy.getByDataHook("ai-settings-validated-badge").should("not.exist") - cy.getByDataHook("ai-settings-provider-openai") - .getByDataHook("ai-settings-provider-status") - .should("be.visible") - .should("contain", "Inactive") - - // When - cy.getByDataHook("ai-settings-save").click() - - // Then - cy.getByDataHook("ai-settings-model-dropdown").should("not.exist") - cy.getByDataHook("ai-chat-button").should("not.exist") - cy.getByDataHook("ai-assistant-settings-button").should( - "contain", - "Configure", - ) - }) - - it("should not provide schema tools when schema access is disabled", () => { - const schemaTools = ["get_tables", "get_table_schema"] - - // Given - interceptTokenValidation("openai", true) - - // When - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - cy.getByDataHook("ai-promo-continue").should("be.visible").click() - cy.getByDataHook("ai-settings-provider-openai").click() - cy.getByDataHook("ai-settings-api-key").type("valid-api-key") - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") - - // When - enable a model, drop permissions to None so schema tools are excluded. - cy.getByDataHook("configure-models-model-row") - .contains("gpt-5-mini") - .click() - cy.getByDataHook("permissions-trigger").click() - cy.getByDataHook("permission-level-none").click() - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - AI chat should be available - cy.get(".toast-success-container").should("be.visible").click() - cy.getByDataHook("ai-chat-button").should("be.visible") - - // When - Open chat and send a message - interceptAIChatRequest("openai", "chatWithoutSchema") - cy.getByDataHook("ai-chat-button").click() - cy.getByDataHook("ai-chat-window").should("be.visible") - cy.getByDataHook("chat-input-textarea").type("Hello, test message") - cy.getByDataHook("chat-send-button").click() - - // Then - Verify request does NOT contain schema tools - cy.wait("@chatWithoutSchema").then((interception) => { - const tools = interception.request.body.tools || [] - const toolNames = tools.map((t) => t.name || t.function?.name) - schemaTools.forEach((schemaTool) => { - expect(toolNames).to.not.include(schemaTool) - }) - }) - - // When - Open settings modal and re-enable schema access - cy.getByDataHook("ai-assistant-settings-button").click() - cy.getByDataHook("permissions-trigger").click() - cy.getByDataHook("permission-level-schema").click() - cy.getByDataHook("ai-settings-save").click() - cy.get(".toast-success-container").should("be.visible").click() - - // When - Send another message - interceptAIChatRequest("openai", "chatWithSchema") - cy.getByDataHook("chat-input-textarea").type("Another test message") - cy.getByDataHook("chat-send-button").click() - - // Then - Verify request DOES contain schema tools - cy.wait("@chatWithSchema").then((interception) => { - const tools = interception.request.body.tools || [] - const toolNames = tools.map((t) => t.name || t.function?.name) - schemaTools.forEach((schemaTool) => { - expect(toolNames).to.include(schemaTool) - }) - }) - }) - - it("should work with multiple providers", () => { - const openaiEnabledModels = ["GPT-5.4", "GPT-5 Mini"] - const anthropicEnabledModels = ["Claude Opus 4.5", "Claude Sonnet 4.5"] - - // Given - Set up OpenAI provider first - interceptTokenValidation("openai", true) - - // When - Complete setup with OpenAI - cy.getByDataHook("ai-assistant-settings-button") - .should("be.visible") - .click() - cy.getByDataHook("ai-promo-continue").should("be.visible").click() - cy.getByDataHook("ai-settings-provider-openai").click() - cy.getByDataHook("ai-settings-api-key").type("valid-openai-key") - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - Should be on step two - cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") - - // When - Enable two OpenAI models - cy.getByDataHook("configure-models-model-row").contains("gpt-5.4").click() - cy.getByDataHook("configure-models-model-row") - .contains("gpt-5-mini") - .click() - - cy.getByDataHook("multi-step-modal-next-button").click() - - // Then - Verify model dropdown shows exactly the enabled OpenAI models - cy.get(".toast-success-container").should("be.visible").click() - cy.getByDataHook("ai-settings-model-dropdown").click() - cy.then(() => { - cy.getByDataHook("ai-settings-model-item").should( - "have.length", - openaiEnabledModels.length, - ) - openaiEnabledModels.forEach((modelLabel) => { - cy.getByDataHook("ai-settings-model-item").contains(modelLabel) - }) - }) - cy.get("body").type("{esc}") // close dropdown - - // When - Open settings and configure Anthropic provider - interceptTokenValidation("anthropic", true) - cy.getByDataHook("ai-assistant-settings-button").click() - - // Then - OpenAI should show Enabled, Anthropic should show Inactive - cy.getByDataHook("ai-settings-provider-openai") - .getByDataHook("ai-settings-provider-status") - .should("contain", "Enabled") - cy.getByDataHook("ai-settings-provider-anthropic") - .getByDataHook("ai-settings-provider-status") - .should("contain", "Inactive") - - // When - Configure Anthropic - cy.getByDataHook("ai-settings-provider-anthropic").click() - cy.getByDataHook("ai-settings-api-key").type("valid-anthropic-key") - cy.getByDataHook("ai-settings-test-api").click() - - // Then - Validation opens Manage Models with the fetched listing - cy.wait("@anthropicValidation") - cy.getByDataHook("manage-models-model-row").should("have.length", 3) - - // When - Enable two Anthropic models and save the picker - cy.getByDataHook("manage-models-model-row") - .contains("Claude Opus 4.5") - .click() - cy.getByDataHook("manage-models-model-row") - .contains("Claude Sonnet 4.5") - .click() - cy.getByDataHook("manage-models-save").click() - - // Then - Anthropic should no longer show Inactive - cy.getByDataHook("ai-settings-provider-anthropic") - .getByDataHook("ai-settings-provider-status") - .should("not.contain", "Inactive") - - // When - Save settings - cy.getByDataHook("ai-settings-save").click() - cy.get(".toast-success-container").should("be.visible").click() - - // Then - Model dropdown should contain models from both providers - cy.getByDataHook("ai-settings-model-dropdown").click() - cy.then(() => { - const allEnabledModels = [ - ...openaiEnabledModels, - ...anthropicEnabledModels, - ] - cy.getByDataHook("ai-settings-model-item").should( - "have.length", - allEnabledModels.length, - ) - allEnabledModels.forEach((modelLabel) => { - cy.getByDataHook("ai-settings-model-item").contains(modelLabel) - }) - }) - - // When - Select first OpenAI model and open chat - cy.then(() => { - cy.getByDataHook("ai-settings-model-item") - .contains(openaiEnabledModels[0]) - .click() - }) - interceptAIChatRequest("openai", "openaiChat") - cy.getByDataHook("ai-chat-button").click() - cy.getByDataHook("ai-chat-window").should("be.visible") - cy.getByDataHook("chat-input-textarea").type("Test message for OpenAI") - cy.getByDataHook("chat-send-button").click() - - // Then - Should intercept OpenAI request - cy.wait("@openaiChat") - - // When - Select first Anthropic model from dropdown - cy.getByDataHook("ai-settings-model-dropdown").click() - cy.then(() => { - cy.getByDataHook("ai-settings-model-item") - .contains(anthropicEnabledModels[0]) - .click() - }) - - // When - Send another message - interceptAIChatRequest("anthropic", "anthropicChat") - cy.getByDataHook("chat-input-textarea").type("Test message for Anthropic") - cy.getByDataHook("chat-send-button").click() - - // Then - Should intercept Anthropic request - cy.wait("@anthropicChat") - }) - }) - describe("ai chat window ergonomics", () => { beforeEach(() => { cy.loadConsoleWithAuth(false, getOpenAIConfiguredSettings()) diff --git a/e2e/tests/console/aiProviderSetup.spec.js b/e2e/tests/console/aiProviderSetup.spec.js new file mode 100644 index 000000000..43f578620 --- /dev/null +++ b/e2e/tests/console/aiProviderSetup.spec.js @@ -0,0 +1,993 @@ +/// + +const { + PROVIDERS, + interceptAIChatRequest, + getAnthropicConfiguredSettings, + createResponse, + createFinalResponseData, + createChatTitleResponse, + isTitleRequest, +} = require("../../utils/aiAssistant") + +const OPENAI_MODELS_URL = "https://api.openai.com/v1/models*" +const ANTHROPIC_MODELS_URL = "https://api.anthropic.com/v1/models*" + +const OPENAI_LISTING = { + object: "list", + data: [ + { id: "gpt-5.4", object: "model", created: 1772000000 }, + { id: "gpt-5-mini", object: "model", created: 1754500000 }, + { id: "gpt-5", object: "model", created: 1754400000 }, + { id: "gpt-5-nano", object: "model", created: 1754300000 }, + { id: "gpt-5-2025-08-06", object: "model", created: 1754400000 }, + { id: "whisper-1", object: "model", created: 1677532384 }, + ], +} + +const ANTHROPIC_LISTING = { + data: [ + { + type: "model", + id: "claude-opus-4-5", + display_name: "Claude Opus 4.5", + created_at: "2025-11-01T00:00:00Z", + }, + { + type: "model", + id: "claude-sonnet-4-5", + display_name: "Claude Sonnet 4.5", + created_at: "2025-09-29T00:00:00Z", + }, + { + type: "model", + id: "claude-haiku-4-5", + display_name: "Claude Haiku 4.5", + created_at: "2025-10-01T00:00:00Z", + }, + ], + has_more: false, + first_id: "claude-opus-4-5", + last_id: "claude-haiku-4-5", +} + +function interceptOpenAIListing(options = {}) { + cy.intercept("GET", OPENAI_MODELS_URL, { + statusCode: 200, + delay: options.delay ?? 200, + body: OPENAI_LISTING, + }).as("openaiListing") +} + +function interceptAnthropicListing(options = {}) { + cy.intercept("GET", ANTHROPIC_MODELS_URL, { + statusCode: 200, + delay: options.delay ?? 200, + body: ANTHROPIC_LISTING, + }).as("anthropicListing") +} + +function readAiSettings(win) { + return JSON.parse(win.localStorage.getItem("ai.assistant.settings")) +} + +/** + * Intercepts AI provider model listing requests. + * Validation now runs through GET /v1/models, so a mocked listing both + * validates the key and feeds the model picker. + * + * The OpenAI listing carries noise (whisper-1) that the picker must filter, + * and `created` timestamps that drive newest-first ordering. + * + * @param {"anthropic" | "openai"} provider - The AI provider to intercept + * @param {boolean} success - If true, returns 200 with a listing; if false, returns 401 + */ +function interceptTokenValidation(provider, success) { + if (provider === "openai") { + if (success) { + cy.intercept("GET", "https://api.openai.com/v1/models*", { + statusCode: 200, + delay: 200, + body: { + object: "list", + data: [ + { id: "gpt-5.4", object: "model", created: 1772000000 }, + { id: "gpt-5-mini", object: "model", created: 1754500000 }, + { id: "gpt-5", object: "model", created: 1754400000 }, + { id: "gpt-5-nano", object: "model", created: 1754300000 }, + { id: "whisper-1", object: "model", created: 1677532384 }, + ], + }, + }).as("openaiValidation") + } else { + cy.intercept("GET", "https://api.openai.com/v1/models*", { + statusCode: 401, + delay: 200, + body: { + error: { + message: + "Incorrect API key provided: ***. You can find your API key at https://platform.openai.com/account/api-keys.", + type: "invalid_request_error", + param: null, + code: "invalid_api_key", + }, + }, + }).as("openaiValidation") + } + } else if (provider === "anthropic") { + if (success) { + cy.intercept("GET", "https://api.anthropic.com/v1/models*", { + statusCode: 200, + delay: 200, + body: { + data: [ + { + type: "model", + id: "claude-opus-4-5", + display_name: "Claude Opus 4.5", + created_at: "2025-11-01T00:00:00Z", + }, + { + type: "model", + id: "claude-sonnet-4-5", + display_name: "Claude Sonnet 4.5", + created_at: "2025-09-29T00:00:00Z", + }, + { + type: "model", + id: "claude-haiku-4-5", + display_name: "Claude Haiku 4.5", + created_at: "2025-10-01T00:00:00Z", + }, + ], + has_more: false, + first_id: "claude-opus-4-5", + last_id: "claude-haiku-4-5", + }, + }).as("anthropicValidation") + } else { + cy.intercept("GET", "https://api.anthropic.com/v1/models*", { + statusCode: 401, + delay: 200, + body: { + type: "error", + error: { + type: "authentication_error", + message: "invalid x-api-key", + }, + request_id: "req_mock_test", + }, + }).as("anthropicValidation") + } + } +} + +describe("ai provider setup flows", () => { + beforeEach(() => { + cy.intercept("POST", PROVIDERS.openai.endpoint, (req) => { + throw new Error( + `Unhandled OpenAI request detected! Request body: ${JSON.stringify(req.body).slice(0, 200)}...`, + ) + }).as("unhandledOpenAI") + + cy.intercept("POST", PROVIDERS.anthropic.endpoint, (req) => { + throw new Error( + `Unhandled Anthropic request detected! Request body: ${JSON.stringify(req.body).slice(0, 200)}...`, + ) + }).as("unhandledAnthropic") + + cy.intercept("GET", "https://api.openai.com/v1/models*", () => { + throw new Error("Unhandled OpenAI model listing request detected!") + }).as("unhandledOpenAIModels") + + cy.intercept("GET", "https://api.anthropic.com/v1/models*", () => { + throw new Error("Unhandled Anthropic model listing request detected!") + }).as("unhandledAnthropicModels") + }) + + it("onboards a first-run OpenAI user from key to a reasoning chat and its fallback", () => { + // Given a fresh console with intercepted OpenAI endpoints + cy.loadConsoleWithAuth() + cy.intercept("GET", OPENAI_MODELS_URL, { + statusCode: 401, + delay: 200, + body: { + error: { + message: "Incorrect API key provided", + type: "invalid_request_error", + param: null, + code: "invalid_api_key", + }, + }, + }).as("openaiListing") + + // When the wizard opens and an invalid key is validated + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-promo-continue").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("invalid-key") + cy.getByDataHook("multi-step-modal-next-button").click() + cy.wait("@openaiListing") + + // Then the field shows an inline error and stays on step one + cy.getByDataHook("ai-settings-api-key-error").should( + "contain", + "Invalid API key", + ) + cy.getByDataHook("ai-settings-modal-step-one").should("be.visible") + + // When the key is corrected and validated + interceptOpenAIListing() + cy.getByDataHook("ai-settings-api-key").clear().type("valid-key") + cy.getByDataHook("multi-step-modal-next-button").click() + cy.wait("@openaiListing") + cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") + + // Then activating with no model shows the footer error bar + cy.getByDataHook("multi-step-modal-next-button").click() + cy.getByDataHook("multi-step-modal-error").should( + "contain", + "Please enable at least one model", + ) + + // When a listed model and a manual model are enabled + cy.getByDataHook("configure-models-model-row").contains("gpt-5.4").click() + cy.getByDataHook("configure-models-manual-model-input").type( + "my-proxy-model", + ) + cy.getByDataHook("configure-models-add-model-button").click() + + // Then the manual model shows as a chip and the error bar is gone on retry + cy.getByDataHook("configure-models-model-chip").should( + "contain", + "my-proxy-model", + ) + + // When reasoning is set to High and the assistant is activated + cy.getByDataHook("reasoning-trigger").click() + cy.getByDataHook("reasoning-level-high").click() + cy.getByDataHook("multi-step-modal-next-button").click() + cy.get(".toast-success-container").should("be.visible").click() + + // Then the persisted settings carry the whole configuration + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.selectedModel).to.equal("gpt-5.4") + expect(settings.providers.openai.enabledModels).to.deep.equal([ + "gpt-5.4", + "my-proxy-model", + ]) + expect(settings.providers.openai.reasoningEffort).to.equal("high") + expect(settings.providers.openai.utilityModel).to.equal("gpt-5-nano") + expect(settings.providers.openai.modelLabels).to.deep.equal({ + "gpt-5.4": "GPT-5.4", + "my-proxy-model": "my-proxy-model", + }) + }) + + // When a chat message is sent + cy.intercept("POST", PROVIDERS.openai.endpoint, (req) => { + if (isTitleRequest("openai", req.body)) { + req.reply(createChatTitleResponse("openai", "Test Chat")) + return + } + req.alias = "reasoningChat" + req.reply( + createResponse( + "openai", + createFinalResponseData("openai", "First answer"), + { streaming: req.body.stream === true }, + ), + ) + }) + cy.getByDataHook("ai-chat-button").click() + cy.getByDataHook("chat-input-textarea").type("hello") + cy.getByDataHook("chat-send-button").click() + + // Then the request leaves the browser with high reasoning effort + cy.wait("@reasoningChat") + .its("request.body.reasoning") + .should("deep.equal", { effort: "high", summary: "auto" }) + + // When the model rejects the reasoning parameter on the next message + cy.intercept("POST", PROVIDERS.openai.endpoint, (req) => { + if (isTitleRequest("openai", req.body)) { + req.reply(createChatTitleResponse("openai", "Test Chat")) + return + } + if (req.body.reasoning) { + req.alias = "rejectedChat" + req.reply({ + statusCode: 400, + body: { + error: { + message: + "Unsupported parameter: 'reasoning.effort' is not supported with this model.", + type: "invalid_request_error", + param: "reasoning.effort", + code: "unsupported_parameter", + }, + }, + }) + return + } + req.alias = "strippedRetry" + req.reply( + createResponse( + "openai", + createFinalResponseData("openai", "Fallback answer"), + { streaming: req.body.stream === true }, + ), + ) + }) + cy.getByDataHook("chat-input-textarea").type("again") + cy.getByDataHook("chat-send-button").click() + + // Then the rejected request is retried without reasoning + cy.wait("@rejectedChat") + cy.wait("@strippedRetry").then((interception) => { + expect(interception.request.body).to.not.have.property("reasoning") + }) + + // And the downgrade is surfaced and persisted as Default + cy.get(".toast-info-container").should( + "contain", + "Reasoning preference changed to Default", + ) + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.providers.openai.reasoningEffort).to.equal("default") + }) + }) + + it("manages the OpenAI provider lifecycle from the settings modal", () => { + // Given a console already configured with Anthropic + cy.loadConsoleWithAuth(false, getAnthropicConfiguredSettings()) + interceptOpenAIListing() + + // When an OpenAI key validates + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("key-one") + cy.getByDataHook("ai-settings-test-api").click() + cy.wait("@openaiListing") + + // Then the picker auto-opens from the validation fetch, with no extra request + cy.getByDataHook("manage-models-model-row").should("have.length", 4) + cy.get("@openaiListing.all").should("have.length", 1) + + // When the picker is cancelled with nothing selected + cy.getByDataHook("manage-models-cancel").click() + + // Then the never-configured provider is dropped back to unvalidated + cy.getByDataHook("ai-settings-test-api").should("be.visible") + cy.getByDataHook("ai-settings-validated-badge").should("not.exist") + + // When validation runs again and Select All is used + cy.getByDataHook("ai-settings-test-api").click() + cy.wait("@openaiListing") + cy.getByDataHook("manage-models-model-row").should("have.length", 4) + cy.getByDataHook("manage-models-show-all").click() + cy.getByDataHook("manage-models-select-all").click() + cy.getByDataHook("manage-models-save").click() + cy.getByDataHook("manage-models-save").should("not.exist") + + // Then only the curated chat models persist — hidden ids stay out + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.providers.openai.enabledModels).to.deep.equal([ + "gpt-5.4", + "gpt-5-mini", + "gpt-5", + "gpt-5-nano", + ]) + }) + + // When the parent modal closes without Save Settings and the page reloads + cy.getByDataHook("ai-settings-cancel").click() + cy.reload() + cy.getEditor().should("be.visible") + + // Then the picks survive and an OpenAI model can be selected + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.getByDataHook("ai-settings-model-item").should("have.length", 6) + cy.getByDataHook("ai-settings-model-item").contains("GPT-5.4").click() + + // When the picker reopens manually it refetches a fresh listing + interceptOpenAIListing() + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-manage-models").click() + cy.wait("@openaiListing") + + // And unticking the selected model keeps the selection on OpenAI + cy.getByDataHook("manage-models-model-row") + .contains("gpt-5.4") + .closest("label") + .find("input[type=checkbox]") + .click() + cy.getByDataHook("manage-models-save").click() + cy.getByDataHook("manage-models-save").should("not.exist") + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.selectedModel).to.equal("gpt-5-mini") + }) + + // When the API key changes to a different one and validates + cy.getByDataHook("ai-settings-edit-api-key").click() + cy.getByDataHook("ai-settings-api-key").should("not.have.attr", "readonly") + cy.getByDataHook("ai-settings-api-key").clear().type("key-two") + cy.getByDataHook("ai-settings-test-api").click() + cy.wait("@openaiListing") + + // Then the old key's picks are cleared in the picker + cy.getByDataHook("manage-models-model-row").should("have.length", 4) + cy.getByDataHook("manage-models-model-row") + .find("input[type=checkbox]:checked") + .should("have.length", 0) + + // And cancelling reverts to the stored working configuration + cy.getByDataHook("manage-models-cancel").click() + cy.getByDataHook("ai-settings-validated-badge").should("be.visible") + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.providers.openai.enabledModels).to.deep.equal([ + "gpt-5-mini", + "gpt-5", + "gpt-5-nano", + ]) + }) + }) + + it("keeps delisted and manual models as removable chips with exact row identity", () => { + // Given stored models: a listed alias, a delisted dated snapshot, a manual id + cy.loadConsoleWithAuth(false, { + "ai.assistant.settings": JSON.stringify({ + selectedModel: "gpt-5.4", + providers: { + openai: { + apiKey: "test-openai-key", + enabledModels: ["gpt-5.4", "gpt-5.4-2026-03-05", "my-proxy-model"], + grantSchemaAccess: false, + }, + }, + }), + }) + interceptOpenAIListing() + + // When Manage Models opens + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-manage-models").click() + cy.wait("@openaiListing") + + // Then the listed alias is checked and the other two are plain chips + cy.getByDataHook("manage-models-model-row") + .contains("gpt-5.4") + .closest("label") + .find("input[type=checkbox]") + .should("be.checked") + cy.getByDataHook("manage-models-model-chip").should("have.length", 2) + cy.getByDataHook("manage-models-model-chip").contains("gpt-5.4-2026-03-05") + cy.getByDataHook("manage-models-model-chip").contains("my-proxy-model") + + // When all models are revealed, alias and dated rows toggle independently + cy.getByDataHook("manage-models-show-all").click() + cy.getByDataHook("manage-models-model-row") + .contains(/^gpt-5$/) + .closest("label") + .find("input[type=checkbox]") + .click() + cy.getByDataHook("manage-models-model-row") + .contains("gpt-5-2025-08-06") + .closest("label") + .find("input[type=checkbox]") + .should("not.be.checked") + + // When the dated chip is removed and the picker saves + cy.getByDataHook("manage-models-model-chip") + .contains("gpt-5.4-2026-03-05") + .closest("[data-hook='manage-models-model-chip']") + .find("[data-hook='manage-models-remove-model']") + .click() + cy.getByDataHook("manage-models-save").click() + cy.getByDataHook("manage-models-save").should("not.exist") + + // Then only the removed id is gone — nothing was dropped silently + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.providers.openai.enabledModels).to.deep.equal([ + "gpt-5.4", + "my-proxy-model", + "gpt-5", + ]) + }) + + // And the dropdown shows derived labels but never invents one for manual ids + cy.getByDataHook("ai-settings-cancel").click() + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.getByDataHook("ai-settings-model-item").contains("GPT-5.4") + cy.getByDataHook("ai-settings-model-item").contains("my-proxy-model") + cy.getByDataHook("ai-settings-model-item") + .contains("My-proxy-model") + .should("not.exist") + }) + + it("survives interruptions: wizard escape, tab switch mid-validation, stale key edits", () => { + // Given a fresh console and a slow listing response + cy.loadConsoleWithAuth() + interceptOpenAIListing({ delay: 1500 }) + + // When the wizard validation is escaped mid-flight + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-promo-continue").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("valid-key") + cy.getByDataHook("multi-step-modal-next-button").click() + cy.get("body").type("{esc}") + cy.wait("@openaiListing") + + // Then reopening lands on a clean step one, not a dead-end step two + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-promo-continue").click() + cy.getByDataHook("ai-settings-modal-step-one").should("be.visible") + cy.getByDataHook("ai-settings-modal-step-two").should("not.exist") + + // When the wizard completes normally to reach the settings modal + interceptOpenAIListing() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("valid-key") + cy.getByDataHook("multi-step-modal-next-button").click() + cy.wait("@openaiListing") + cy.getByDataHook("configure-models-model-row").contains("gpt-5.4").click() + cy.getByDataHook("multi-step-modal-next-button").click() + cy.get(".toast-success-container").should("be.visible").click() + + // And an Anthropic validation starts while the tab switches to OpenAI + interceptAnthropicListing({ delay: 1500 }) + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-settings-provider-anthropic").click() + cy.getByDataHook("ai-settings-api-key").type("anthropic-key") + cy.getByDataHook("ai-settings-test-api").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.wait("@anthropicListing") + + // Then the tab switch aborted the validation: no picker, nothing validated + cy.getByDataHook("manage-models-model-row").should("not.exist") + cy.getByDataHook("ai-settings-provider-anthropic").click() + cy.getByDataHook("ai-settings-test-api") + .should("be.visible") + .should("not.be.disabled") + + // And validating without switching opens the Anthropic picker and saves there + interceptAnthropicListing() + cy.getByDataHook("ai-settings-test-api").click() + cy.wait("@anthropicListing") + cy.getByDataHook("manage-models-model-row").should("have.length", 3) + cy.get("[role=dialog]").should("contain", "Enable the Anthropic models") + cy.getByDataHook("manage-models-model-row") + .contains("Claude Haiku 4.5") + .click() + cy.getByDataHook("manage-models-save").click() + cy.getByDataHook("manage-models-save").should("not.exist") + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.providers.anthropic.enabledModels).to.deep.equal([ + "claude-haiku-4-5", + ]) + expect(settings.providers.openai.enabledModels).to.deep.equal(["gpt-5.4"]) + }) + + // When a validation response arrives for a key that was already edited + interceptOpenAIListing({ delay: 1500 }) + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-edit-api-key").click() + cy.getByDataHook("ai-settings-api-key").should("not.have.attr", "readonly") + cy.getByDataHook("ai-settings-api-key").clear().type("key-a") + cy.getByDataHook("ai-settings-test-api").click() + cy.getByDataHook("ai-settings-edit-api-key").click() + cy.getByDataHook("ai-settings-api-key").should("not.have.attr", "readonly") + cy.getByDataHook("ai-settings-api-key").clear() + cy.getByDataHook("ai-settings-api-key").type("key-b-changed") + cy.wait("@openaiListing") + + // Then the stale response is discarded: no badge, no picker, ready to validate + cy.getByDataHook("ai-settings-validated-badge").should("not.exist") + cy.getByDataHook("manage-models-model-row").should("not.exist") + cy.getByDataHook("ai-settings-test-api") + .should("be.visible") + .should("not.be.disabled") + }) + + describe("onboarding and settings", () => { + beforeEach(() => { + cy.loadConsoleWithAuth() + }) + + it("should display ai assistant promo", () => { + // When + cy.getByDataHook("ai-assistant-settings-button") + .should("be.visible") + .click() + + // Then + cy.getByDataHook("ai-promo-modal").should("be.visible") + + // When + cy.getByDataHook("ai-promo-close").should("be.visible").click() + + // Then + cy.getByDataHook("ai-promo-modal").should("not.exist") + + // When + cy.getByDataHook("ai-assistant-settings-button") + .should("be.visible") + .click() + cy.getByDataHook("ai-promo-continue").should("be.visible").click() + + // Then + cy.getByDataHook("ai-settings-modal-step-one").should("be.visible") + }) + + it("should handle invalid api key", () => { + // When + cy.getByDataHook("ai-assistant-settings-button") + .should("be.visible") + .click() + cy.getByDataHook("ai-promo-continue").should("be.visible").click() + + // Then + cy.getByDataHook("ai-settings-modal-step-one").should("be.visible") + // API key input is hidden until a provider is selected + cy.getByDataHook("ai-settings-api-key").should("not.exist") + + // When - select Anthropic + cy.getByDataHook("ai-settings-provider-anthropic").click() + + // Then - API key input appears + cy.getByDataHook("ai-settings-api-key") + .should("be.visible") + .should("have.attr", "placeholder", "Enter Anthropic API key") + + // When - switch to OpenAI + cy.getByDataHook("ai-settings-provider-openai").click() + + // Then + cy.getByDataHook("ai-settings-api-key") + .should("be.visible") + .should("have.attr", "placeholder", "Enter OpenAI API key") + ;["anthropic", "openai"].forEach((provider) => { + // Given + interceptTokenValidation(provider, false) + + // When + cy.getByDataHook(`ai-settings-provider-${provider}`).click() + + // Then + cy.getByDataHook("ai-settings-api-key") + .should("be.visible") + .should( + "have.attr", + "placeholder", + `Enter ${provider === "anthropic" ? "Anthropic" : "OpenAI"} API key`, + ) + .should("be.empty") + + // When + cy.getByDataHook("ai-settings-api-key").type("invalid-api-key") + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then + cy.getByDataHook("multi-step-modal-next-button") + .should("be.disabled") + .should("contain", "Validating...") + + // When + cy.wait(`@${provider}Validation`) + + // Then + cy.getByDataHook("ai-settings-api-key-error").should("be.visible") + }) + }) + + it("should show ai buttons after setup is completed", () => { + // Given + interceptTokenValidation("openai", true) + + // When + cy.getByDataHook("ai-assistant-settings-button") + .should("be.visible") + .click() + cy.getByDataHook("ai-promo-continue").should("be.visible").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("valid-api-key") + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then - step two shows the filtered listing, nothing preselected + cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") + cy.getByDataHook("configure-models-model-row").should("have.length", 4) + + // When - enable two models and activate + cy.getByDataHook("configure-models-model-row").contains("gpt-5.4").click() + cy.getByDataHook("configure-models-model-row") + .contains("gpt-5-mini") + .click() + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then + cy.getByDataHook("ai-assistant-settings-button").should( + "contain", + "AI Settings", + ) + cy.getByDataHook("ai-chat-button").should("be.visible") + cy.getByDataHook("ai-settings-model-dropdown").should("be.visible") + + // When / Then — selecting a model closes the dropdown (handleModelSelect + // calls setDropdownActive(false)), so re-open it for each model and + // re-query the item just before clicking; otherwise the list detaches the + // node as it settles/closes and cy.click() hits a stale element. + ;[0, 1].forEach((index) => { + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.getByDataHook("ai-settings-model-item").should("be.visible") + + cy.getByDataHook("ai-settings-model-item") + .eq(index) + .find("[data-hook='ai-settings-model-item-label']") + .invoke("text") + .then((text) => { + const label = text.trim() + cy.getByDataHook("ai-settings-model-item").eq(index).click() + cy.getByDataHook("ai-settings-model-dropdown").should( + "contain", + label, + ) + }) + }) + + // When + cy.typeQuery("SELECT 1;") + + // Then + cy.getAIIconInLine(1).should("be.visible") + + // When + cy.getByDataHook("ai-assistant-settings-button").click() + + // Then + cy.getByDataHook("ai-settings-validated-badge") + .should("be.visible") + .should("contain", "Validated") + cy.getByDataHook("ai-settings-provider-openai") + .getByDataHook("ai-settings-provider-status") + .should("be.visible") + .should("contain", "Enabled") + + cy.getByDataHook("ai-settings-provider-anthropic") + .getByDataHook("ai-settings-provider-status") + .should("be.visible") + .should("contain", "Inactive") + + // When + cy.getByDataHook("ai-settings-remove-provider").scrollIntoView() + cy.getByDataHook("ai-settings-remove-provider") + .should("be.visible") + .click() + + // Then + cy.getByDataHook("ai-settings-validated-badge").should("not.exist") + cy.getByDataHook("ai-settings-provider-openai") + .getByDataHook("ai-settings-provider-status") + .should("be.visible") + .should("contain", "Inactive") + + // When + cy.getByDataHook("ai-settings-save").click() + + // Then + cy.getByDataHook("ai-settings-model-dropdown").should("not.exist") + cy.getByDataHook("ai-chat-button").should("not.exist") + cy.getByDataHook("ai-assistant-settings-button").should( + "contain", + "Configure", + ) + }) + + it("should not provide schema tools when schema access is disabled", () => { + const schemaTools = ["get_tables", "get_table_schema"] + + // Given + interceptTokenValidation("openai", true) + + // When + cy.getByDataHook("ai-assistant-settings-button") + .should("be.visible") + .click() + cy.getByDataHook("ai-promo-continue").should("be.visible").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("valid-api-key") + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then + cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") + + // When - enable a model, drop permissions to None so schema tools are excluded. + cy.getByDataHook("configure-models-model-row") + .contains("gpt-5-mini") + .click() + cy.getByDataHook("permissions-trigger").click() + cy.getByDataHook("permission-level-none").click() + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then - AI chat should be available + cy.get(".toast-success-container").should("be.visible").click() + cy.getByDataHook("ai-chat-button").should("be.visible") + + // When - Open chat and send a message + interceptAIChatRequest("openai", "chatWithoutSchema") + cy.getByDataHook("ai-chat-button").click() + cy.getByDataHook("ai-chat-window").should("be.visible") + cy.getByDataHook("chat-input-textarea").type("Hello, test message") + cy.getByDataHook("chat-send-button").click() + + // Then - Verify request does NOT contain schema tools + cy.wait("@chatWithoutSchema").then((interception) => { + const tools = interception.request.body.tools || [] + const toolNames = tools.map((t) => t.name || t.function?.name) + schemaTools.forEach((schemaTool) => { + expect(toolNames).to.not.include(schemaTool) + }) + }) + + // When - Open settings modal and re-enable schema access + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("permissions-trigger").click() + cy.getByDataHook("permission-level-schema").click() + cy.getByDataHook("ai-settings-save").click() + cy.get(".toast-success-container").should("be.visible").click() + + // When - Send another message + interceptAIChatRequest("openai", "chatWithSchema") + cy.getByDataHook("chat-input-textarea").type("Another test message") + cy.getByDataHook("chat-send-button").click() + + // Then - Verify request DOES contain schema tools + cy.wait("@chatWithSchema").then((interception) => { + const tools = interception.request.body.tools || [] + const toolNames = tools.map((t) => t.name || t.function?.name) + schemaTools.forEach((schemaTool) => { + expect(toolNames).to.include(schemaTool) + }) + }) + }) + + it("should work with multiple providers", () => { + const openaiEnabledModels = ["GPT-5.4", "GPT-5 Mini"] + const anthropicEnabledModels = ["Claude Opus 4.5", "Claude Sonnet 4.5"] + + // Given - Set up OpenAI provider first + interceptTokenValidation("openai", true) + + // When - Complete setup with OpenAI + cy.getByDataHook("ai-assistant-settings-button") + .should("be.visible") + .click() + cy.getByDataHook("ai-promo-continue").should("be.visible").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-api-key").type("valid-openai-key") + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then - Should be on step two + cy.getByDataHook("ai-settings-modal-step-two").should("be.visible") + + // When - Enable two OpenAI models + cy.getByDataHook("configure-models-model-row").contains("gpt-5.4").click() + cy.getByDataHook("configure-models-model-row") + .contains("gpt-5-mini") + .click() + + cy.getByDataHook("multi-step-modal-next-button").click() + + // Then - Verify model dropdown shows exactly the enabled OpenAI models + cy.get(".toast-success-container").should("be.visible").click() + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.then(() => { + cy.getByDataHook("ai-settings-model-item").should( + "have.length", + openaiEnabledModels.length, + ) + openaiEnabledModels.forEach((modelLabel) => { + cy.getByDataHook("ai-settings-model-item").contains(modelLabel) + }) + }) + cy.get("body").type("{esc}") // close dropdown + + // When - Open settings and configure Anthropic provider + interceptTokenValidation("anthropic", true) + cy.getByDataHook("ai-assistant-settings-button").click() + + // Then - OpenAI should show Enabled, Anthropic should show Inactive + cy.getByDataHook("ai-settings-provider-openai") + .getByDataHook("ai-settings-provider-status") + .should("contain", "Enabled") + cy.getByDataHook("ai-settings-provider-anthropic") + .getByDataHook("ai-settings-provider-status") + .should("contain", "Inactive") + + // When - Configure Anthropic + cy.getByDataHook("ai-settings-provider-anthropic").click() + cy.getByDataHook("ai-settings-api-key").type("valid-anthropic-key") + cy.getByDataHook("ai-settings-test-api").click() + + // Then - Validation opens Manage Models with the fetched listing + cy.wait("@anthropicValidation") + cy.getByDataHook("manage-models-model-row").should("have.length", 3) + + // When - Enable two Anthropic models and save the picker + cy.getByDataHook("manage-models-model-row") + .contains("Claude Opus 4.5") + .click() + cy.getByDataHook("manage-models-model-row") + .contains("Claude Sonnet 4.5") + .click() + cy.getByDataHook("manage-models-save").click() + + // Then - Anthropic should no longer show Inactive + cy.getByDataHook("ai-settings-provider-anthropic") + .getByDataHook("ai-settings-provider-status") + .should("not.contain", "Inactive") + + // When - Save settings (the picker's own save may still show its toast) + cy.getByDataHook("ai-settings-save").click() + cy.get(".toast-success-container") + .should("be.visible") + .click({ multiple: true }) + + // Then - Model dropdown should contain models from both providers + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.then(() => { + const allEnabledModels = [ + ...openaiEnabledModels, + ...anthropicEnabledModels, + ] + cy.getByDataHook("ai-settings-model-item").should( + "have.length", + allEnabledModels.length, + ) + allEnabledModels.forEach((modelLabel) => { + cy.getByDataHook("ai-settings-model-item").contains(modelLabel) + }) + }) + + // When - Select first OpenAI model and open chat + cy.then(() => { + cy.getByDataHook("ai-settings-model-item") + .contains(openaiEnabledModels[0]) + .click() + }) + interceptAIChatRequest("openai", "openaiChat") + cy.getByDataHook("ai-chat-button").click() + cy.getByDataHook("ai-chat-window").should("be.visible") + cy.getByDataHook("chat-input-textarea").type("Test message for OpenAI") + cy.getByDataHook("chat-send-button").click() + + // Then - Should intercept OpenAI request + cy.wait("@openaiChat") + + // When - Select first Anthropic model from dropdown + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.then(() => { + cy.getByDataHook("ai-settings-model-item") + .contains(anthropicEnabledModels[0]) + .click() + }) + + // When - Send another message + interceptAIChatRequest("anthropic", "anthropicChat") + cy.getByDataHook("chat-input-textarea").type("Test message for Anthropic") + cy.getByDataHook("chat-send-button").click() + + // Then - Should intercept Anthropic request + cy.wait("@anthropicChat") + }) + }) +}) diff --git a/e2e/utils/aiAssistant.js b/e2e/utils/aiAssistant.js index c3bbecbe1..430c32e76 100644 --- a/e2e/utils/aiAssistant.js +++ b/e2e/utils/aiAssistant.js @@ -1163,7 +1163,42 @@ function createMultiTurnFlow(config) { } } +/** + * Intercepts AI chat requests with a default test response. + * + * @param {"anthropic" | "openai"} provider - The AI provider to intercept + * @param {string} [alias] - Optional custom alias for the intercept + * @param {number} [delay=200] - Delay in milliseconds + * @param {Object} [options] - Options + * @param {boolean} [options.streaming=true] - Whether to use streaming response + */ +function interceptAIChatRequest( + provider, + alias, + delay = 200, + options = { streaming: true }, +) { + const aliasName = alias || `${provider}ChatRequest` + const endpoint = PROVIDERS[provider].endpoint + const { streaming = true } = options + + const responseData = createFinalResponseData( + provider, + "Test response explanation", + ) + + cy.intercept("POST", endpoint, (req) => { + if (isTitleRequest(provider, req.body)) { + req.reply(createChatTitleResponse(provider, "Test Chat")) + return + } + req.alias = aliasName + req.reply(createResponse(provider, responseData, { streaming, delay })) + }) +} + module.exports = { + interceptAIChatRequest, PROVIDERS, CUSTOM_PROVIDER_DEFAULTS, getOpenAIConfiguredSettings, diff --git a/src/components/MultiStepModal/index.tsx b/src/components/MultiStepModal/index.tsx index c68d0bf23..351756484 100644 --- a/src/components/MultiStepModal/index.tsx +++ b/src/components/MultiStepModal/index.tsx @@ -1,7 +1,14 @@ -import React, { ReactNode, useState, createContext, useContext } from "react" +import React, { + ReactNode, + useState, + useRef, + createContext, + useContext, +} from "react" import * as RadixDialog from "@radix-ui/react-dialog" import styled, { css } from "styled-components" import { ArrowLeft } from "../icons" +import { ValidationNotice } from "../ValidationNotice" import { Overlay } from "../Overlay" import { Box } from "../Box" import { Button } from "../Button" @@ -140,13 +147,6 @@ const FooterButtons = styled(Box).attrs({ width: 100%; ` -const ValidationError = styled(Text)` - color: ${({ theme }) => theme.color.statusDanger}; - font-size: 1.3rem; - text-align: right; - width: 100%; -` - const CancelButton = styled(Button)` flex: 1; padding: 1.1rem 1.2rem; @@ -186,7 +186,6 @@ type MultiStepModalProps = { canProceed?: (stepIndex: number) => boolean | Promise completeButtonText?: string onStepChange?: (stepIndex: number, direction: "next" | "previous") => void - showValidationError?: boolean } export const MultiStepModal = ({ @@ -199,18 +198,20 @@ export const MultiStepModal = ({ canProceed, completeButtonText = "Complete", onStepChange, - showValidationError = true, }: MultiStepModalProps) => { const [currentStep, setCurrentStep] = useState(0) const [validationError, setValidationError] = useState(null) const [isValidating, setIsValidating] = useState(false) + const sessionRef = useRef(0) + const handleOpenChange = (isOpen: boolean) => { if (!isOpen && onCancel) { onCancel() } onOpenChange?.(isOpen) if (!isOpen) { + sessionRef.current += 1 setCurrentStep(0) setValidationError(null) setIsValidating(false) @@ -225,19 +226,21 @@ export const MultiStepModal = ({ } if (currentStepData?.validate) { + const session = sessionRef.current setValidationError(null) setIsValidating(true) try { const validationResult = await currentStepData.validate() + if (session !== sessionRef.current) return if (typeof validationResult === "string") { setValidationError(validationResult) return } else if (validationResult === false) { - setValidationError("Validation failed") return } } catch (error) { + if (session !== sessionRef.current) return const errorMessage = error instanceof Error ? error.message : "Validation failed" setValidationError(errorMessage) @@ -323,10 +326,12 @@ export const MultiStepModal = ({ ? steps[currentStep]?.content() : steps[currentStep]?.content} + {validationError && ( + + {validationError} + + )} - {showValidationError && validationError && ( - {validationError} - )} {error && ( - - {error} - + + + {error} + + )} Stored locally in your browser and never sent to QuestDB @@ -511,6 +514,7 @@ export const ConfigurationModal = ({ onOpenChange, }: ConfigurationModalProps) => { const { aiAssistantSettings, updateSettings } = useLocalStorage() + const closeCountRef = useRef(0) const [selectedProvider, setSelectedProvider] = useState( null, ) @@ -580,25 +584,14 @@ export const ConfigurationModal = ({ selectedModel: models[0], providers: { ...aiAssistantSettings.providers, - [selectedProvider]: { + [selectedProvider]: buildProviderSettings({ apiKey, enabledModels: models, - grantSchemaAccess: permissions.grantSchemaAccess, - read: permissions.read, - write: permissions.write, - ...(metadata && Object.keys(metadata.modelLabels).length > 0 - ? { modelLabels: metadata.modelLabels } - : {}), - ...(metadata?.utilityModel - ? { utilityModel: metadata.utilityModel } - : {}), - ...(metadata?.reasoningModels?.length - ? { reasoningModels: metadata.reasoningModels } - : {}), - ...(reasoningEffortLevel === "high" - ? { reasoningEffort: "high" as const } - : {}), - }, + permissions, + modelLabels: metadata?.modelLabels, + utilityModel: metadata?.utilityModel, + reasoningEffort: reasoningEffortLevel, + }), }, } @@ -629,8 +622,10 @@ export const ConfigurationModal = ({ apiKey, aiAssistantSettings, ) + const session = closeCountRef.current try { const listing = await provider.listModels() + if (session !== closeCountRef.current) return false setProviderListing(listing) setError(null) void trackEvent(ConsoleEvent.AI_CONFIGURATION_VALIDATE) @@ -642,7 +637,7 @@ export const ConfigurationModal = ({ ? "Invalid API key" : classified.message setError(errorMessage) - return errorMessage + return false } }, [selectedProvider, apiKey, aiAssistantSettings]) @@ -669,6 +664,7 @@ export const ConfigurationModal = ({ ) const handleModalClose = useCallback(() => { + closeCountRef.current += 1 setSelectedProvider(null) setApiKey("") setError(null) @@ -795,7 +791,6 @@ export const ConfigurationModal = ({ onComplete={handleComplete} canProceed={canProceed} completeButtonText="Activate Assistant" - showValidationError={false} /> {customProviderModalOpen && ( utilityModel?: string - reasoningModels?: string[] } type BuiltinModelsRef = { @@ -126,140 +125,142 @@ type BuiltinModelsContentProps = { providerId: string apiKey: string enabledModels: string[] + initialListing?: ProviderModel[] onLoadingChange: (loading: boolean) => void } const BuiltinModelsContent = forwardRef< BuiltinModelsRef, BuiltinModelsContentProps ->(({ providerId, apiKey, enabledModels, onLoadingChange }, ref) => { - const [listing, setListing] = useState(null) - const [fetchFailed, setFetchFailed] = useState(false) - const [selectedModels, setSelectedModels] = useState([]) - const [unavailableModels, setUnavailableModels] = useState([]) - const [manualInput, setManualInput] = useState("") - const [isLoading, setIsLoading] = useState(true) +>( + ( + { providerId, apiKey, enabledModels, initialListing, onLoadingChange }, + ref, + ) => { + const [listing, setListing] = useState( + initialListing ?? null, + ) + const [fetchFailed, setFetchFailed] = useState(false) + const [selectedModels, setSelectedModels] = useState( + initialListing ? enabledModels : [], + ) + const [manualInput, setManualInput] = useState("") + const [isLoading, setIsLoading] = useState(!initialListing) - const isOpenAi = BUILTIN_PROVIDERS[providerId]?.type === "openai" - const pickerModels = listing - ? isOpenAi - ? filterOpenAiChatModels(listing) - : sortModelsNewestFirst(listing) - : [] - const hiddenModels = - listing && isOpenAi - ? sortModelsNewestFirst( - listing.filter((m) => !pickerModels.some((p) => p.id === m.id)), - ) - : undefined + const isOpenAi = BUILTIN_PROVIDERS[providerId]?.type === "openai" + const pickerModels = listing + ? isOpenAi + ? filterOpenAiChatModels(listing) + : sortModelsNewestFirst(listing) + : [] + const hiddenModels = + listing && isOpenAi + ? sortModelsNewestFirst( + listing.filter((m) => !pickerModels.some((p) => p.id === m.id)), + ) + : undefined - const selectionWithPending = () => { - const pending = manualInput.trim() - return pending && !selectedModels.includes(pending) - ? [...selectedModels, pending] - : [...selectedModels] - } + const selectionWithPending = () => { + const pending = manualInput.trim() + return pending && !selectedModels.includes(pending) + ? [...selectedModels, pending] + : [...selectedModels] + } - useImperativeHandle( - ref, - () => ({ - getResult: () => { - if (!listing) return null - const models = selectionWithPending() - return { - enabledModels: models, - ...buildListingMetadata(providerId, listing, models), - } - }, - validate: () => { - if (!listing) return "Could not fetch models from the provider" - if (selectionWithPending().length === 0) - return "Enable at least one model" - return true - }, - }), - [listing, selectedModels, manualInput, providerId], - ) + useImperativeHandle( + ref, + () => ({ + getResult: () => { + if (!listing) return null + const models = selectionWithPending() + return { + enabledModels: models, + ...buildListingMetadata(providerId, listing, models), + } + }, + validate: () => { + if (!listing) return "Could not fetch models from the provider" + if (selectionWithPending().length === 0) + return "Enable at least one model" + return true + }, + }), + [listing, selectedModels, manualInput, providerId], + ) - useEffect(() => { - let cancelled = false + useEffect(() => { + let cancelled = false - const doFetch = async () => { - setIsLoading(true) - onLoadingChange(true) - try { - const provider = createProviderByType( - BUILTIN_PROVIDERS[providerId].type, - providerId, - apiKey, - ) - const models = await provider.listModels() - if (cancelled) return - setListing(models) - setSelectedModels( - enabledModels.filter((id) => - models.some((m) => matchesListedModel(id, m.id)), - ), - ) - setUnavailableModels( - enabledModels.filter( - (id) => !models.some((m) => matchesListedModel(id, m.id)), - ), - ) - } catch { - if (cancelled) return - setFetchFailed(true) - } finally { - if (!cancelled) { - setIsLoading(false) + const doFetch = async () => { + if (initialListing) { onLoadingChange(false) + return + } + onLoadingChange(true) + try { + const provider = createProviderByType( + BUILTIN_PROVIDERS[providerId].type, + providerId, + apiKey, + ) + const models = await provider.listModels() + if (cancelled) return + setListing(models) + setSelectedModels(enabledModels) + } catch { + if (cancelled) return + setFetchFailed(true) + } finally { + if (!cancelled) { + setIsLoading(false) + onLoadingChange(false) + } } } - } - void doFetch() - return () => { - cancelled = true + void doFetch() + return () => { + cancelled = true + } + }, []) + + if (isLoading) { + return ( + + + + + + ) } - }, []) - if (isLoading) { - return ( - - - - - - ) - } + if (fetchFailed) { + return ( + + + Could not fetch models from the provider. Check your API key and + connection, then try again. + + + ) + } - if (fetchFailed) { return ( - - Could not fetch models from the provider. Check your API key and - connection, then try again. - + model.label ?? formatModelLabel(model.id)} + onSelectionChange={setSelectedModels} + onManualInputChange={setManualInput} + /> ) - } - - return ( - - model.label ?? formatModelLabel(model.id)} - onSelectionChange={setSelectedModels} - onManualInputChange={setManualInput} - /> - - ) -}) + }, +) BuiltinModelsContent.displayName = "BuiltinModelsContent" @@ -277,6 +278,7 @@ type ManageModelsModalProps = { variant: "builtin" apiKey: string enabledModels: string[] + initialListing?: ProviderModel[] onSave: (providerId: string, result: BuiltinModelsResult) => void } ) @@ -364,13 +366,19 @@ export const ManageModelsModal = (props: ManageModelsModalProps) => { providerId={providerId} apiKey={props.apiKey} enabledModels={props.enabledModels} + initialListing={props.initialListing} onLoadingChange={setModelsLoading} /> )} - + {error ? ( + + {error} + + ) : ( + + )} - {error && {error}} theme.color.contentSecondary}; ` -const UnavailableRow = styled(Box).attrs({ - gap: "0.8rem", - align: "center", -})` - padding: 0.6rem 0.8rem; - font-size: 1.4rem; - color: ${({ theme }) => theme.color.contentPrimary}; -` - -const UnavailableHint = styled(Text)` - font-size: 1.2rem; - color: ${({ theme }) => theme.color.statusWarning}; - margin-left: auto; -` - const ShowAllButton = styled(TextButton)` font-size: 1.3rem; align-self: flex-start; @@ -146,7 +131,6 @@ export type ModelPickerProps = { listedModels: ProviderModel[] hiddenModels?: ProviderModel[] selectedModels: string[] - unavailableModels?: string[] manualInput: string dataHookPrefix: string labelFor?: (model: ProviderModel) => string @@ -158,39 +142,33 @@ export const ModelPicker = ({ listedModels, hiddenModels, selectedModels, - unavailableModels, manualInput, dataHookPrefix, labelFor, onSelectionChange, onManualInputChange, }: ModelPickerProps) => { - const theme = useTheme() const [showAll, setShowAll] = useState(false) const visibleModels = showAll && hiddenModels?.length ? sortModelsNewestFirst([...listedModels, ...hiddenModels]) : listedModels - const isRowChecked = (rowId: string) => - selectedModels.some((selected) => matchesListedModel(selected, rowId)) - const isListedAnywhere = (selected: string) => - listedModels.some((m) => matchesListedModel(selected, m.id)) || - (hiddenModels?.some((m) => matchesListedModel(selected, m.id)) ?? false) - const manualModels = selectedModels.filter((m) => !isListedAnywhere(m)) + const isRowChecked = (rowId: string) => selectedModels.includes(rowId) + const manualModels = selectedModels.filter( + (selected) => !visibleModels.some((m) => m.id === selected), + ) const handleToggleRow = (rowId: string) => { if (isRowChecked(rowId)) { - onSelectionChange( - selectedModels.filter((s) => !matchesListedModel(s, rowId)), - ) + onSelectionChange(selectedModels.filter((s) => s !== rowId)) } else { onSelectionChange([...selectedModels, rowId]) } } const handleSelectAll = () => { - const unchecked = visibleModels + const unchecked = listedModels .filter((m) => !isRowChecked(m.id)) .map((m) => m.id) onSelectionChange([...selectedModels, ...unchecked]) @@ -198,9 +176,7 @@ export const ModelPicker = ({ const handleDeselectAll = () => { onSelectionChange( - selectedModels.filter( - (s) => !visibleModels.some((m) => matchesListedModel(s, m.id)), - ), + selectedModels.filter((s) => !visibleModels.some((m) => m.id === s)), ) } @@ -239,23 +215,7 @@ export const ModelPicker = ({ - - {unavailableModels?.map((model) => ( - - - {model} - - Removed by the provider. Save removes it. - - - ))} + {visibleModels.map((model) => { const label = labelFor ? labelFor(model) : model.id return ( @@ -288,6 +248,7 @@ export const ModelPicker = ({ onManualInputChange(e.target.value)} diff --git a/src/components/SetupAIAssistant/ReasoningSection.tsx b/src/components/SetupAIAssistant/ReasoningSection.tsx index c5171760b..16adf22db 100644 --- a/src/components/SetupAIAssistant/ReasoningSection.tsx +++ b/src/components/SetupAIAssistant/ReasoningSection.tsx @@ -7,20 +7,11 @@ export type ReasoningEffortLevel = "default" | "high" type Option = { level: ReasoningEffortLevel label: string - hint: string } const OPTIONS: Option[] = [ - { - level: "default", - label: "Default", - hint: "Each model uses its own default reasoning level.", - }, - { - level: "high", - label: "High", - hint: "Maximum thinking — slower, better for hard questions.", - }, + { level: "default", label: "Default" }, + { level: "high", label: "High" }, ] const Field = styled.div` @@ -57,10 +48,10 @@ export const ReasoningSection: React.FC = ({ @@ -72,7 +63,6 @@ export const ReasoningSection: React.FC = ({ {opt.label} diff --git a/src/components/SetupAIAssistant/SettingsModal.tsx b/src/components/SetupAIAssistant/SettingsModal.tsx index 87cd9db15..afa7e589e 100644 --- a/src/components/SetupAIAssistant/SettingsModal.tsx +++ b/src/components/SetupAIAssistant/SettingsModal.tsx @@ -1,4 +1,4 @@ -import React, { useState, useCallback, useMemo, useRef } from "react" +import React, { useState, useCallback, useEffect, useMemo, useRef } from "react" import styled, { useTheme } from "styled-components" import * as RadixDialog from "@radix-ui/react-dialog" import { Dialog } from "../Dialog" @@ -23,11 +23,13 @@ import { getAllModelOptions, getApiKey, makeCustomModelValue, - parseModelValue, + stripModelNamespace, formatModelLabel, + buildProviderSettings, BUILTIN_PROVIDERS, type ModelOption, type ProviderId, + type ProviderModel, getNextModel, getProviderName, } from "../../utils/ai" @@ -486,14 +488,6 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { undefined, ), ) - const [reasoningModels, setReasoningModels] = useState< - Record - >(() => - initializeProviderState( - (provider) => aiAssistantSettings.providers?.[provider]?.reasoningModels, - undefined, - ), - ) const [reasoningEffort, setReasoningEffort] = useState< Record >(() => @@ -545,7 +539,8 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { const inputRef = useRef(null) const [customProviderModalOpen, setCustomProviderModalOpen] = useState(false) - const [manageModelsModalOpen, setManageModelsModalOpen] = useState(false) + const [manageModelsProvider, setManageModelsProvider] = + useState(null) const [localCustomProviders, setLocalCustomProviders] = useState< Record @@ -562,15 +557,45 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { [aiAssistantSettings, localCustomProviders], ) - const handleProviderSelect = useCallback((provider: ProviderId) => { - setSelectedProvider(provider) - setValidationErrors((prev) => ({ ...prev, [provider]: null })) + const handleProviderSelect = useCallback( + (provider: ProviderId) => { + if (provider !== selectedProvider) { + abortValidation(selectedProvider) + } + setSelectedProvider(provider) + setValidationErrors((prev) => ({ ...prev, [provider]: null })) + }, + [selectedProvider], + ) + + const validationTokenRef = useRef>({}) + const [validationListings, setValidationListings] = useState< + Record + >({}) + const mountedRef = useRef(true) + + useEffect(() => { + return () => { + mountedRef.current = false + } }, []) + const abortValidation = (provider: ProviderId) => { + validationTokenRef.current[provider] = + (validationTokenRef.current[provider] ?? 0) + 1 + setValidationState((prev) => + prev[provider] === "validating" ? { ...prev, [provider]: "idle" } : prev, + ) + setValidationListings((prev) => ({ ...prev, [provider]: undefined })) + } + const handleApiKeyChange = useCallback( (provider: ProviderId, value: string) => { + validationTokenRef.current[provider] = + (validationTokenRef.current[provider] ?? 0) + 1 setApiKeys((prev) => ({ ...prev, [provider]: value })) setValidationErrors((prev) => ({ ...prev, [provider]: null })) + setValidationState((prev) => ({ ...prev, [provider]: "idle" })) if (validatedApiKeys[provider]) { setValidatedApiKeys((prev) => ({ ...prev, [provider]: false })) @@ -593,17 +618,32 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { setValidationState((prev) => ({ ...prev, [provider]: "validating" })) setValidationErrors((prev) => ({ ...prev, [provider]: null })) + const token = validationTokenRef.current[provider] ?? 0 + const isStale = () => + !mountedRef.current || + (validationTokenRef.current[provider] ?? 0) !== token const isBuiltin = !!BUILTIN_PROVIDERS[provider] try { const aiProvider = createProvider(provider, apiKey, localSettings) - await aiProvider.listModels() + const listing = await aiProvider.listModels() + if (isStale()) return + if (isBuiltin) { + setValidationListings((prev) => ({ ...prev, [provider]: listing })) + } setValidationState((prev) => ({ ...prev, [provider]: "validated" })) setValidatedApiKeys((prev) => ({ ...prev, [provider]: true })) setValidationErrors((prev) => ({ ...prev, [provider]: null })) + const storedKey = localSettings.providers?.[provider]?.apiKey + if (isBuiltin && apiKey !== storedKey) { + setEnabledModels((prev) => ({ ...prev, [provider]: [] })) + setModelLabels((prev) => ({ ...prev, [provider]: {} })) + setUtilityModels((prev) => ({ ...prev, [provider]: undefined })) + } if (isBuiltin) { - setManageModelsModalOpen(true) + setManageModelsProvider(provider) } } catch (err) { + if (isStale()) return const aiProvider = createProvider(provider, apiKey, localSettings) const classified = aiProvider.classifyError(err, () => {}) if (!isBuiltin && classified.type !== "invalid_key") { @@ -649,25 +689,14 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { if (validatedApiKeys[provider] || isCustom) { const perms = permissions[provider] const labels = modelLabels[provider] - updatedProviders[provider] = { + updatedProviders[provider] = buildProviderSettings({ apiKey: apiKeys[provider] ?? "", enabledModels: enabledModels[provider], - grantSchemaAccess: perms.grantSchemaAccess, - read: perms.read, - write: perms.write, - ...(labels && Object.keys(labels).length > 0 - ? { modelLabels: labels } - : {}), - ...(utilityModels[provider] - ? { utilityModel: utilityModels[provider] } - : {}), - ...(reasoningModels[provider]?.length - ? { reasoningModels: reasoningModels[provider] } - : {}), - ...(reasoningEffort[provider] === "high" - ? { reasoningEffort: "high" as const } - : {}), - } + permissions: perms, + modelLabels: labels, + utilityModel: utilityModels[provider], + reasoningEffort: reasoningEffort[provider], + }) } else { delete updatedProviders[provider] } @@ -707,6 +736,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { updatedSettings.selectedModel, enabledModels, updatedSettings, + aiAssistantSettings, ) updatedSettings.selectedModel = nextModel || undefined @@ -721,7 +751,6 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { enabledModels, modelLabels, utilityModels, - reasoningModels, reasoningEffort, permissions, validatedApiKeys, @@ -735,6 +764,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { const handleRemoveProvider = useCallback( (providerId: ProviderId) => { + abortValidation(providerId) const isCustom = !BUILTIN_PROVIDERS[providerId] void trackEvent(ConsoleEvent.AI_SETTINGS_PROVIDER_REMOVE, { isCustom, @@ -758,7 +788,6 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { setEnabledModels((prev) => ({ ...prev, [providerId]: [] })) setModelLabels((prev) => ({ ...prev, [providerId]: {} })) setUtilityModels((prev) => ({ ...prev, [providerId]: undefined })) - setReasoningModels((prev) => ({ ...prev, [providerId]: undefined })) setReasoningEffort((prev) => ({ ...prev, [providerId]: "default" })) setIsInputFocused((prev) => ({ ...prev, [providerId]: false })) @@ -906,8 +935,45 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { [aiAssistantSettings, enabledModels, localCustomProviders, updateSettings], ) + const builtinModelsSavedRef = useRef(false) + + const handleBuiltinModelsOpenChange = useCallback( + (nextOpen: boolean) => { + if (nextOpen) return + const provider = manageModelsProvider + setManageModelsProvider(null) + const cancelled = !builtinModelsSavedRef.current + builtinModelsSavedRef.current = false + if (!provider) return + setValidationListings((prev) => ({ ...prev, [provider]: undefined })) + if (!cancelled || (enabledModels[provider]?.length ?? 0) > 0) return + abortValidation(provider) + const stored = aiAssistantSettings.providers?.[provider] + if (stored?.enabledModels?.length) { + setApiKeys((prev) => ({ ...prev, [provider]: stored.apiKey })) + setEnabledModels((prev) => ({ + ...prev, + [provider]: stored.enabledModels, + })) + setModelLabels((prev) => ({ + ...prev, + [provider]: stored.modelLabels ?? {}, + })) + setUtilityModels((prev) => ({ + ...prev, + [provider]: stored.utilityModel, + })) + return + } + setValidatedApiKeys((prev) => ({ ...prev, [provider]: false })) + setValidationState((prev) => ({ ...prev, [provider]: "idle" })) + }, + [aiAssistantSettings, enabledModels, manageModelsProvider], + ) + const handleBuiltinModelsSave = useCallback( (providerId: string, result: BuiltinModelsResult) => { + builtinModelsSavedRef.current = true setEnabledModels((prev) => ({ ...prev, [providerId]: result.enabledModels, @@ -917,12 +983,48 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { ...prev, [providerId]: result.utilityModel, })) - setReasoningModels((prev) => ({ - ...prev, - [providerId]: result.reasoningModels, - })) + + const perms = permissions[providerId] + const updatedSettings: AiAssistantSettings = { + ...aiAssistantSettings, + providers: { + ...aiAssistantSettings.providers, + [providerId]: buildProviderSettings({ + apiKey: apiKeys[providerId] ?? "", + enabledModels: result.enabledModels, + permissions: perms, + modelLabels: result.modelLabels, + utilityModel: result.utilityModel, + reasoningEffort: reasoningEffort[providerId], + }), + }, + } + const persistedEnabledModels = Object.fromEntries( + Object.entries(updatedSettings.providers).map( + ([provider, providerSettings]) => [ + provider, + providerSettings?.enabledModels ?? [], + ], + ), + ) + updatedSettings.selectedModel = + getNextModel( + updatedSettings.selectedModel, + persistedEnabledModels, + updatedSettings, + aiAssistantSettings, + ) || undefined + + updateSettings(StoreKey.AI_ASSISTANT_SETTINGS, updatedSettings) + toast.success("Model preferences updated") }, - [], + [ + aiAssistantSettings, + apiKeys, + permissions, + reasoningEffort, + updateSettings, + ], ) const currentProviderValidated = validatedApiKeys[selectedProvider] @@ -946,7 +1048,8 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { ) const labelForModel = (provider: ProviderId, value: string) => { - if (!BUILTIN_PROVIDERS[provider]) return parseModelValue(value).rawModel + if (!BUILTIN_PROVIDERS[provider]) + return stripModelNamespace(value, provider) return modelLabels[provider]?.[value] ?? formatModelLabel(value) } @@ -972,7 +1075,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { return ( <> @@ -1202,7 +1305,9 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { setManageModelsModalOpen(true)} + onClick={() => + setManageModelsProvider(selectedProvider) + } > Manage models @@ -1316,26 +1421,31 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { )} /> )} - {manageModelsModalOpen && - isCustomProvider && - localCustomProviders[selectedProvider] && ( + {manageModelsProvider && + !BUILTIN_PROVIDERS[manageModelsProvider] && + localCustomProviders[manageModelsProvider] && ( { + if (!nextOpen) setManageModelsProvider(null) + }} + providerId={manageModelsProvider} + definition={localCustomProviders[manageModelsProvider]} onSave={handleManageModelsSave} /> )} - {manageModelsModalOpen && !isCustomProvider && ( + {manageModelsProvider && BUILTIN_PROVIDERS[manageModelsProvider] && ( )} diff --git a/src/components/ValidationNotice/index.tsx b/src/components/ValidationNotice/index.tsx new file mode 100644 index 000000000..b4f70f6f0 --- /dev/null +++ b/src/components/ValidationNotice/index.tsx @@ -0,0 +1,37 @@ +import React, { ReactNode } from "react" +import styled from "styled-components" +import { WarningIcon } from "@phosphor-icons/react" + +const Notice = styled.div` + display: flex; + flex-shrink: 0; + align-items: center; + gap: 0.6rem; + width: 100%; + padding: 0.6rem 2.4rem; + background: ${({ theme }) => theme.color.statusDangerSurface}; + border-top: 0.1rem solid ${({ theme }) => theme.color.interactionNeutral}; + color: ${({ theme }) => theme.color.contentPrimary}; + font-size: 1.3rem; + line-height: 1.3; + + > svg { + flex-shrink: 0; + color: ${({ theme }) => theme.color.statusDanger}; + } +` + +type ValidationNoticeProps = { + children: ReactNode + dataHook: string +} + +export const ValidationNotice = ({ + children, + dataHook, +}: ValidationNoticeProps) => ( + + + {children} + +) diff --git a/src/providers/LocalStorageProvider/index.tsx b/src/providers/LocalStorageProvider/index.tsx index 36fab08b0..d2c73875a 100644 --- a/src/providers/LocalStorageProvider/index.tsx +++ b/src/providers/LocalStorageProvider/index.tsx @@ -50,6 +50,7 @@ import { RunWithSelectionMode, } from "./types" import { reconcileSettings } from "../../utils/ai/settings" +import { onReasoningUnsupported } from "../../utils/ai/reasoningFallback" export const DEFAULT_AI_ASSISTANT_SETTINGS: AiAssistantSettings = { providers: {}, @@ -381,6 +382,23 @@ export const LocalStorageProvider = ({ [refreshSettings], ) + useEffect( + () => + onReasoningUnsupported((providerId) => { + const settings = getAiAssistantSettings() + const providerSettings = settings.providers[providerId] + if (providerSettings?.reasoningEffort !== "high") return + updateSettings(StoreKey.AI_ASSISTANT_SETTINGS, { + ...settings, + providers: { + ...settings.providers, + [providerId]: { ...providerSettings, reasoningEffort: "default" }, + }, + }) + }), + [updateSettings], + ) + const value = useMemo( () => ({ editorCol, diff --git a/src/providers/LocalStorageProvider/types.ts b/src/providers/LocalStorageProvider/types.ts index 2e440d4a4..55a000006 100644 --- a/src/providers/LocalStorageProvider/types.ts +++ b/src/providers/LocalStorageProvider/types.ts @@ -10,7 +10,6 @@ export type ProviderSettings = { modelLabels?: Record utilityModel?: string reasoningEffort?: "default" | "high" - reasoningModels?: string[] } export type CustomProviderDefinition = { diff --git a/src/utils/ai/anthropicProvider.ts b/src/utils/ai/anthropicProvider.ts index b1677a8f9..cd2e139dc 100644 --- a/src/utils/ai/anthropicProvider.ts +++ b/src/utils/ai/anthropicProvider.ts @@ -8,7 +8,7 @@ import type { StreamingCallback, TokenUsage, } from "./aiAssistant" -import { parseModelValue } from "./settings" +import { stripModelNamespace } from "./settings" import type { ProviderId } from "./settings" import { type AIProvider, @@ -130,10 +130,6 @@ function toAnthropicTools(tools: ToolDefinition[]): AnthropicTool[] { })) } -function toAnthropicModel(model: string): string { - return parseModelValue(model).rawModel -} - async function createAnthropicMessage( anthropic: Anthropic, params: Omit & { @@ -500,7 +496,7 @@ export function createAnthropicProvider( const toolContext: ToolExecutionContext = incomingToolContext ?? {} - const resolvedModel = toAnthropicModel(model) + const resolvedModel = stripModelNamespace(model, providerId) const messageParams: Parameters[1] = { model: resolvedModel, @@ -583,7 +579,7 @@ export function createAnthropicProvider( async generateTitle({ model, prompt }) { try { const message = await createAnthropicMessage(anthropic, { - model: toAnthropicModel(model), + model: stripModelNamespace(model, providerId), messages: [{ role: "user", content: prompt }], max_tokens: 100, }) @@ -609,7 +605,7 @@ export function createAnthropicProvider( let text = "" const stream = anthropic.messages.stream( { - model: toAnthropicModel(model), + model: stripModelNamespace(model, providerId), max_tokens: 64_000, messages: [{ role: "user", content: userMessage }], system: systemPrompt, @@ -640,7 +636,7 @@ export function createAnthropicProvider( const nativeMessages = toNativeMessages(messages) const response = await anthropic.messages.countTokens({ - model: toAnthropicModel(model), + model: stripModelNamespace(model, providerId), system: systemPrompt, messages: nativeMessages, }) diff --git a/src/utils/ai/index.ts b/src/utils/ai/index.ts index 234c11d1d..21770b947 100644 --- a/src/utils/ai/index.ts +++ b/src/utils/ai/index.ts @@ -37,8 +37,9 @@ export { getUtilityModel, getProviderContextWindow, getApiKey, + buildProviderSettings, makeCustomModelValue, - parseModelValue, + stripModelNamespace, isAiAssistantConfigured, canUseAiAssistant, hasSchemaAccess, @@ -52,10 +53,8 @@ export type { CustomProviderDefinition, } from "./settings" export { - computeReasoningModels, filterOpenAiChatModels, formatModelLabel, - matchesListedModel, resolveUtilityModel, sortModelsNewestFirst, UTILITY_MODEL_TIERS, diff --git a/src/utils/ai/modelCatalog.test.ts b/src/utils/ai/modelCatalog.test.ts index 8ff1f11b4..c85b00ec8 100644 --- a/src/utils/ai/modelCatalog.test.ts +++ b/src/utils/ai/modelCatalog.test.ts @@ -1,10 +1,7 @@ import { describe, it, expect } from "vitest" import { - computeReasoningModels, filterOpenAiChatModels, formatModelLabel, - isReasoningModel, - matchesListedModel, resolveUtilityModel, stripDateSuffix, UTILITY_MODEL_TIERS, @@ -31,26 +28,6 @@ describe("stripDateSuffix", () => { }) }) -describe("matchesListedModel", () => { - it("matches exact ids", () => { - expect(matchesListedModel("gpt-5.4", "gpt-5.4")).toBe(true) - }) - - it("matches a stored alias against its dated listing id", () => { - expect( - matchesListedModel("claude-sonnet-4-5", "claude-sonnet-4-5-20250929"), - ).toBe(true) - }) - - it("does not match a stored dated id against a different dated id", () => { - expect(matchesListedModel("gpt-4-0613", "gpt-4-1106")).toBe(false) - }) - - it("does not match unrelated ids", () => { - expect(matchesListedModel("gpt-5.4", "gpt-5.4-mini")).toBe(false) - }) -}) - describe("formatModelLabel", () => { it("derives labels from OpenAI ids", () => { expect(formatModelLabel("gpt-5-mini")).toBe("GPT-5 Mini") @@ -142,7 +119,7 @@ describe("resolveUtilityModel", () => { ) }) - it("collapses dated ids to their alias and keeps the newest", () => { + it("dedupes dated variants per alias and returns the listed id verbatim", () => { const listing = [ model("claude-haiku-4-5-20251001", 100), model("claude-haiku-4-6", 200), @@ -153,31 +130,16 @@ describe("resolveUtilityModel", () => { ) }) + it("keeps a dated winner's exact listed id", () => { + const listing = [model("claude-haiku-4-5-20251001", 100)] + expect(resolveUtilityModel(listing, UTILITY_MODEL_TIERS.anthropic)).toBe( + "claude-haiku-4-5-20251001", + ) + }) + it("returns null when no tier matches", () => { expect( resolveUtilityModel([model("gpt-5.4", 100)], UTILITY_MODEL_TIERS.openai), ).toBeNull() }) }) - -describe("reasoning gate", () => { - it("gates models created on or after the gpt-5 launch", () => { - const listing = [ - model("gpt-5", AUG_2025), - model("gpt-4.1", JUL_2025), - model("gpt-6", AUG_2025 + 1_000_000), - ] - expect(computeReasoningModels(listing)).toEqual(["gpt-5", "gpt-6"]) - }) - - it("treats models without a timestamp as ungated", () => { - expect(computeReasoningModels([{ id: "mystery-model" }])).toEqual([]) - }) - - it("matches enabled aliases against gated dated ids", () => { - const gated = ["gpt-5.4-2026-03-05", "gpt-5.4"] - expect(isReasoningModel("gpt-5.4", gated)).toBe(true) - expect(isReasoningModel("gpt-4.1", gated)).toBe(false) - expect(isReasoningModel("gpt-4.1", undefined)).toBe(false) - }) -}) diff --git a/src/utils/ai/modelCatalog.ts b/src/utils/ai/modelCatalog.ts index c872e2b8f..749f051c6 100644 --- a/src/utils/ai/modelCatalog.ts +++ b/src/utils/ai/modelCatalog.ts @@ -38,11 +38,6 @@ const OPENAI_NON_CHAT_TOKENS = [ export const stripDateSuffix = (id: string): string => id.replace(DATE_SUFFIX, "") -export const matchesListedModel = ( - storedId: string, - listedId: string, -): boolean => storedId === listedId || stripDateSuffix(listedId) === storedId - const isNumericToken = (token: string): boolean => /^[\d.]+$/.test(token) export const formatModelLabel = (id: string): string => { @@ -104,7 +99,7 @@ export const resolveUtilityModel = ( const alias = stripDateSuffix(model.id) const existing = byAlias.get(alias) if (!existing || (model.created ?? 0) > (existing.created ?? 0)) { - byAlias.set(alias, { ...model, id: alias }) + byAlias.set(alias, model) } } for (const tier of tiers) { @@ -115,13 +110,3 @@ export const resolveUtilityModel = ( } return null } - -export const computeReasoningModels = (models: ProviderModel[]): string[] => - models.filter((m) => (m.created ?? 0) >= GPT5_LAUNCH_START).map((m) => m.id) - -export const isReasoningModel = ( - modelId: string, - reasoningModels: string[] | undefined, -): boolean => - reasoningModels?.some((listed) => matchesListedModel(modelId, listed)) ?? - false diff --git a/src/utils/ai/openaiChatCompletionsProvider.ts b/src/utils/ai/openaiChatCompletionsProvider.ts index 47a7c1738..ec0de616b 100644 --- a/src/utils/ai/openaiChatCompletionsProvider.ts +++ b/src/utils/ai/openaiChatCompletionsProvider.ts @@ -9,9 +9,8 @@ import type { StreamingCallback, TokenUsage, } from "./aiAssistant" -import { parseModelValue } from "./settings" +import { stripModelNamespace } from "./settings" import type { ProviderId } from "./settings" -import { isReasoningModel } from "./modelCatalog" import { type AIProvider, type ExecuteFlowParams, @@ -36,7 +35,6 @@ import { classifyOpenAIError, countTokensFromNativePayload, isOpenAINonRetryableError, - isReasoningRejection, } from "./openaiShared" import { createHeaderFilteredFetch, @@ -291,28 +289,6 @@ async function executeRequest( params: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming, streaming?: StreamingCallback, abortSignal?: AbortSignal, -): Promise { - try { - return await executeRequestOnce(openai, params, streaming, abortSignal) - } catch (error) { - if (params.reasoning_effort && isReasoningRejection(error)) { - const { reasoning_effort: _reasoningEffort, ...withoutReasoning } = params - return executeRequestOnce( - openai, - withoutReasoning, - streaming, - abortSignal, - ) - } - throw error - } -} - -async function executeRequestOnce( - openai: OpenAI, - params: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming, - streaming?: StreamingCallback, - abortSignal?: AbortSignal, ): Promise { if (streaming) { const accumulated = await createChatCompletionStreaming( @@ -382,7 +358,6 @@ export function createOpenAIChatCompletionsProvider( baseURL?: string contextWindow?: number isCustom?: boolean - reasoning?: { effort: "high"; models: string[] } }, ): AIProvider { const isCustom = options?.isCustom ?? false @@ -399,19 +374,6 @@ export function createOpenAIChatCompletionsProvider( const contextWindow = options?.contextWindow ?? 400_000 - const toRequestProps = ( - model: string, - ): { model: string; reasoning_effort?: OpenAI.ReasoningEffort } => { - const rawModel = parseModelValue(model).rawModel - return { - model: rawModel, - ...(options?.reasoning && - isReasoningModel(rawModel, options.reasoning.models) - ? { reasoning_effort: options.reasoning.effort } - : {}), - } - } - return { id: providerId, contextWindow, @@ -451,7 +413,7 @@ export function createOpenAIChatCompletionsProvider( const toolContext: ToolExecutionContext = incomingToolContext ?? {} const baseParams = { - ...toRequestProps(model), + model: stripModelNamespace(model, providerId), tools: openaiTools, } @@ -593,7 +555,7 @@ export function createOpenAIChatCompletionsProvider( async generateTitle({ model, prompt }) { try { const response = await openai.chat.completions.create({ - model: parseModelValue(model).rawModel, + model: stripModelNamespace(model, providerId), messages: [{ role: "user", content: prompt }], ...(isCustom ? {} : { max_completion_tokens: 100 }), }) @@ -616,7 +578,7 @@ export function createOpenAIChatCompletionsProvider( }) { let text = "" const summaryParams = { - ...toRequestProps(model), + model: stripModelNamespace(model, providerId), messages: [ { role: "system" as const, content: systemPrompt }, { role: "user" as const, content: userMessage }, @@ -626,23 +588,10 @@ export function createOpenAIChatCompletionsProvider( const requestOptions = abortSignal ? ([{ signal: abortSignal }] as const) : ([] as const) - let stream - try { - stream = await openai.chat.completions.create( - summaryParams, - ...requestOptions, - ) - } catch (error) { - if (!summaryParams.reasoning_effort || !isReasoningRejection(error)) { - throw error - } - const { reasoning_effort: _reasoningEffort, ...withoutReasoning } = - summaryParams - stream = await openai.chat.completions.create( - withoutReasoning, - ...requestOptions, - ) - } + const stream = await openai.chat.completions.create( + summaryParams, + ...requestOptions, + ) for await (const chunk of stream) { const delta = chunk.choices[0]?.delta?.content if (delta) { diff --git a/src/utils/ai/openaiProvider.fallback.test.ts b/src/utils/ai/openaiProvider.fallback.test.ts new file mode 100644 index 000000000..66bf451e4 --- /dev/null +++ b/src/utils/ai/openaiProvider.fallback.test.ts @@ -0,0 +1,164 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest" +import { createOpenAIProvider } from "./openaiProvider" +import { onReasoningUnsupported } from "./reasoningFallback" + +const { createMock, toastInfoMock } = vi.hoisted(() => ({ + createMock: vi.fn(), + toastInfoMock: vi.fn(), +})) + +vi.mock("openai", async (importOriginal) => { + const actual = await importOriginal() + class MockOpenAI { + static APIError = actual.default.APIError + static APIUserAbortError = actual.default.APIUserAbortError + responses = { create: createMock } + } + return { default: MockOpenAI } +}) + +vi.mock("../../components/Toast", () => ({ + toast: { info: toastInfoMock }, +})) + +const reasoningRejection = async () => { + const OpenAI = (await import("openai")).default + const message = + "Unsupported parameter: 'reasoning.effort' is not supported with this model." + return new OpenAI.APIError( + 400, + { message, param: "reasoning.effort", type: "invalid_request_error" }, + message, + undefined, + ) +} + +const requestBodies = () => + createMock.mock.calls.map( + (call: unknown[]) => call[0] as Record, + ) + +const textStream = (text: string) => + (async function* () { + await Promise.resolve() + yield { type: "response.output_text.delta", delta: text } + })() + +describe("openai reasoning fallback", () => { + let unsupportedProviders: string[] + let unregister: () => void + + beforeEach(() => { + createMock.mockReset() + toastInfoMock.mockReset() + unsupportedProviders = [] + unregister = onReasoningUnsupported((providerId) => + unsupportedProviders.push(providerId), + ) + }) + + afterEach(() => { + unregister() + }) + + it("sends high effort, then strips reasoning and retries once on a rejection", async () => { + // Given a model that rejects the reasoning parameter + createMock + .mockRejectedValueOnce(await reasoningRejection()) + .mockReturnValueOnce(textStream("summary")) + const provider = createOpenAIProvider("sk-test", "openai", { + reasoning: { effort: "high" }, + }) + + // When generating a summary + const text = await provider.generateSummary({ + model: "gpt-4o", + systemPrompt: "sys", + userMessage: "user", + }) + + // Then the first request carried reasoning and the retry dropped it + expect(text).toBe("summary") + expect(createMock).toHaveBeenCalledTimes(2) + expect(requestBodies()[0].reasoning).toEqual({ + effort: "high", + summary: "auto", + }) + expect("reasoning" in requestBodies()[1]).toBe(false) + + // And the downgrade was surfaced and reported exactly once + expect(toastInfoMock).toHaveBeenCalledWith( + "Reasoning preference changed to Default", + ) + expect(unsupportedProviders).toEqual(["openai"]) + }) + + it("stops sending reasoning on later requests after a rejection", async () => { + createMock + .mockRejectedValueOnce(await reasoningRejection()) + .mockReturnValue(textStream("again")) + const provider = createOpenAIProvider("sk-test", "openai", { + reasoning: { effort: "high" }, + }) + await provider.generateSummary({ + model: "gpt-4o", + systemPrompt: "sys", + userMessage: "user", + }) + + // When a second request runs on the same provider instance + await provider.generateSummary({ + model: "gpt-4o", + systemPrompt: "sys", + userMessage: "user", + }) + + // Then it goes out once, without reasoning, and nothing is re-reported + expect(createMock).toHaveBeenCalledTimes(3) + expect("reasoning" in requestBodies()[2]).toBe(false) + expect(toastInfoMock).toHaveBeenCalledTimes(1) + expect(unsupportedProviders).toEqual(["openai"]) + }) + + it("rethrows unrelated 400s without retrying", async () => { + const OpenAI = (await import("openai")).default + const message = "Item of type 'reasoning' was provided without its pair." + createMock.mockRejectedValueOnce( + new OpenAI.APIError( + 400, + { message, param: null, type: "invalid_request_error" }, + message, + undefined, + ), + ) + const provider = createOpenAIProvider("sk-test", "openai", { + reasoning: { effort: "high" }, + }) + + await expect( + provider.generateSummary({ + model: "gpt-4o", + systemPrompt: "sys", + userMessage: "user", + }), + ).rejects.toThrow() + + expect(createMock).toHaveBeenCalledTimes(1) + expect(toastInfoMock).not.toHaveBeenCalled() + expect(unsupportedProviders).toEqual([]) + }) + + it("sends no reasoning at all when effort is default", async () => { + createMock.mockReturnValueOnce(textStream("plain")) + const provider = createOpenAIProvider("sk-test", "openai") + + await provider.generateSummary({ + model: "gpt-5.4", + systemPrompt: "sys", + userMessage: "user", + }) + + expect(createMock).toHaveBeenCalledTimes(1) + expect("reasoning" in requestBodies()[0]).toBe(false) + }) +}) diff --git a/src/utils/ai/openaiProvider.ts b/src/utils/ai/openaiProvider.ts index 57f0ecd4b..4164f1792 100644 --- a/src/utils/ai/openaiProvider.ts +++ b/src/utils/ai/openaiProvider.ts @@ -5,9 +5,10 @@ import type { StatusCallback, StreamingCallback, } from "./aiAssistant" -import { parseModelValue } from "./settings" +import { stripModelNamespace } from "./settings" import type { ProviderId } from "./settings" -import { isReasoningModel } from "./modelCatalog" +import { reportReasoningUnsupported } from "./reasoningFallback" +import { toast } from "../../components/Toast" import { type AIProvider, type ExecuteFlowParams, @@ -133,7 +134,6 @@ async function createOpenAIResponseStreaming( ...params, stream: true, store: false, - include: ["reasoning.encrypted_content"], } as OpenAI.Responses.ResponseCreateParamsStreaming, { signal: abortSignal }, ) @@ -258,26 +258,15 @@ function getOpenAIText(response: OpenAI.Responses.Response): { return { type: "text", message: "" } } -async function createResponseWithReasoningFallback( - openai: OpenAI, +function stripReasoning( params: OpenAI.Responses.ResponseCreateParamsNonStreaming, - streaming?: StreamingCallback, - abortSignal?: AbortSignal, -): Promise { - const run = async (p: OpenAI.Responses.ResponseCreateParamsNonStreaming) => - streaming - ? (await createOpenAIResponseStreaming(openai, p, streaming, abortSignal)) - .response - : openai.responses.create(p) - try { - return await run(params) - } catch (error) { - if (params.reasoning && isReasoningRejection(error)) { - const { reasoning: _reasoning, ...withoutReasoning } = params - return run(withoutReasoning) - } - throw error - } +): OpenAI.Responses.ResponseCreateParamsNonStreaming { + const { + reasoning: _reasoning, + include: _include, + ...withoutReasoning + } = params + return withoutReasoning } export function createOpenAIProvider( @@ -287,7 +276,7 @@ export function createOpenAIProvider( baseURL?: string contextWindow?: number isCustom?: boolean - reasoning?: { effort: "high"; models: string[] } + reasoning?: { effort: "high" } }, ): AIProvider { const isCustom = options?.isCustom ?? false @@ -304,14 +293,21 @@ export function createOpenAIProvider( const contextWindow = options?.contextWindow ?? 400_000 + let reasoningUnsupported = false + + const markReasoningUnsupported = () => { + if (reasoningUnsupported) return + reasoningUnsupported = true + toast.info("Reasoning preference changed to Default") + reportReasoningUnsupported(providerId) + } + const toRequestProps = ( model: string, ): { model: string; reasoning?: OpenAI.Reasoning } => { - const rawModel = parseModelValue(model).rawModel return { - model: rawModel, - ...(options?.reasoning && - isReasoningModel(rawModel, options.reasoning.models) + model: stripModelNamespace(model, providerId), + ...(options?.reasoning && !reasoningUnsupported ? { reasoning: { effort: options.reasoning.effort, @@ -322,6 +318,36 @@ export function createOpenAIProvider( } } + const createResponseWithReasoningFallback = async ( + params: OpenAI.Responses.ResponseCreateParamsNonStreaming, + streaming?: StreamingCallback, + abortSignal?: AbortSignal, + ): Promise => { + const run = async (p: OpenAI.Responses.ResponseCreateParamsNonStreaming) => + streaming + ? ( + await createOpenAIResponseStreaming( + openai, + p, + streaming, + abortSignal, + ) + ).response + : openai.responses.create(p) + const effectiveParams = reasoningUnsupported + ? stripReasoning(params) + : params + try { + return await run(effectiveParams) + } catch (error) { + if (effectiveParams.reasoning && isReasoningRejection(error)) { + markReasoningUnsupported() + return run(stripReasoning(params)) + } + throw error + } + } + return { id: providerId, contextWindow, @@ -368,7 +394,6 @@ export function createOpenAIProvider( } as OpenAI.Responses.ResponseCreateParamsNonStreaming let lastResponse = await createResponseWithReasoningFallback( - openai, requestParams, streaming, abortSignal, @@ -471,7 +496,6 @@ export function createOpenAIProvider( } as OpenAI.Responses.ResponseCreateParamsNonStreaming lastResponse = await createResponseWithReasoningFallback( - openai, loopRequestParams, streaming, abortSignal, @@ -522,7 +546,7 @@ export function createOpenAIProvider( async generateTitle({ model, prompt }) { try { const response = await openai.responses.create({ - model: parseModelValue(model).rawModel, + model: stripModelNamespace(model, providerId), input: [{ role: "user", content: prompt }], max_output_tokens: 100, }) @@ -560,6 +584,7 @@ export function createOpenAIProvider( if (!summaryParams.reasoning || !isReasoningRejection(error)) { throw error } + markReasoningUnsupported() const { reasoning: _reasoning, ...withoutReasoning } = summaryParams stream = await openai.responses.create( withoutReasoning, diff --git a/src/utils/ai/openaiShared.test.ts b/src/utils/ai/openaiShared.test.ts new file mode 100644 index 000000000..7afaf06d5 --- /dev/null +++ b/src/utils/ai/openaiShared.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect } from "vitest" +import OpenAI from "openai" +import { isReasoningRejection } from "./openaiShared" + +const apiError = (status: number, param: string | null, message: string) => + new OpenAI.APIError( + status, + { message, param, type: "invalid_request_error" }, + message, + undefined, + ) + +describe("isReasoningRejection", () => { + it("matches a 400 whose param names reasoning", () => { + // Given the Responses API rejects reasoning on a non-reasoning model + const responsesError = apiError( + 400, + "reasoning.effort", + "Unsupported parameter: 'reasoning.effort' is not supported with this model.", + ) + const chatError = apiError( + 400, + "reasoning_effort", + "Unsupported parameter: 'reasoning_effort' is not supported with this model.", + ) + + // Then both wire formats are recognized + expect(isReasoningRejection(responsesError)).toBe(true) + expect(isReasoningRejection(chatError)).toBe(true) + }) + + it("ignores a 400 that only mentions reasoning in its message", () => { + // Given an unrelated 400 whose message contains the word "reasoning" + const error = apiError( + 400, + null, + "Item of type 'reasoning' was provided without its required following item.", + ) + + // Then it is not treated as a reasoning rejection + expect(isReasoningRejection(error)).toBe(false) + }) + + it("ignores non-400 statuses and non-API errors", () => { + expect( + isReasoningRejection(apiError(429, "reasoning.effort", "rate limited")), + ).toBe(false) + expect(isReasoningRejection(new Error("reasoning failed"))).toBe(false) + }) +}) diff --git a/src/utils/ai/openaiShared.ts b/src/utils/ai/openaiShared.ts index f99b55a5e..d8158b512 100644 --- a/src/utils/ai/openaiShared.ts +++ b/src/utils/ai/openaiShared.ts @@ -5,11 +5,7 @@ import { StreamingError, RefusalError, MaxTokensError } from "./shared" export function isReasoningRejection(error: unknown): boolean { if (!(error instanceof OpenAI.APIError) || error.status !== 400) return false - const param = (error as { param?: string | null }).param - return ( - (typeof param === "string" && param.includes("reasoning")) || - error.message.toLowerCase().includes("reasoning") - ) + return typeof error.param === "string" && error.param.startsWith("reasoning") } let tiktokenEncoder: Tiktoken | null = null diff --git a/src/utils/ai/reasoningFallback.ts b/src/utils/ai/reasoningFallback.ts new file mode 100644 index 000000000..ec25d3413 --- /dev/null +++ b/src/utils/ai/reasoningFallback.ts @@ -0,0 +1,16 @@ +type ReasoningUnsupportedHandler = (providerId: string) => void + +let handler: ReasoningUnsupportedHandler | null = null + +export const onReasoningUnsupported = ( + nextHandler: ReasoningUnsupportedHandler, +): (() => void) => { + handler = nextHandler + return () => { + if (handler === nextHandler) handler = null + } +} + +export const reportReasoningUnsupported = (providerId: string): void => { + handler?.(providerId) +} diff --git a/src/utils/ai/registry.ts b/src/utils/ai/registry.ts index 85a9a21e6..2415d8f3e 100644 --- a/src/utils/ai/registry.ts +++ b/src/utils/ai/registry.ts @@ -10,23 +10,17 @@ type ProviderOptions = { baseURL?: string contextWindow?: number isCustom?: boolean - reasoning?: { effort: "high"; models: string[] } + reasoning?: { effort: "high" } } const reasoningOptions = ( providerId: ProviderId, settings?: AiAssistantSettings, ): Pick => { - const providerSettings = settings?.providers?.[providerId] - if ( - providerSettings?.reasoningEffort !== "high" || - !providerSettings.reasoningModels?.length - ) { - return {} - } - return { - reasoning: { effort: "high", models: providerSettings.reasoningModels }, - } + if (BUILTIN_PROVIDERS[providerId]?.type !== "openai") return {} + return settings?.providers?.[providerId]?.reasoningEffort === "high" + ? { reasoning: { effort: "high" } } + : {} } export function createProvider( diff --git a/src/utils/ai/settings.test.ts b/src/utils/ai/settings.test.ts index e1565bc40..d7cc86b2e 100644 --- a/src/utils/ai/settings.test.ts +++ b/src/utils/ai/settings.test.ts @@ -7,6 +7,7 @@ import { getNextModel, getUtilityModel, providerForModel, + buildListingMetadata, } from "./settings" import type { AiAssistantSettings } from "../../providers/LocalStorageProvider/types" @@ -304,8 +305,38 @@ describe("providerForModel", () => { expect(providerForModel("gpt-5-mini", settings)).toBeNull() }) - it("parses namespaced custom values without settings lookup", () => { - expect(providerForModel("custom-1:llm-a")).toBe("custom-1") + it("treats a colon prefix as a namespace only when that custom provider exists", () => { + // Given a custom provider named custom-1 + const settings = makeSettings({ + customProviders: { + "custom-1": { + type: "openai-chat-completions", + name: "Test", + baseURL: "http://localhost:11434/v1", + contextWindow: 100_000, + models: ["llm-a"], + }, + }, + }) + // Then only its own prefix resolves as a namespace + expect(providerForModel("custom-1:llm-a", settings)).toBe("custom-1") + expect(providerForModel("custom-1:llm-a")).toBeNull() + }) + + it("routes colon-containing listed ids to the built-in provider that enabled them", () => { + // Given an OpenAI fine-tune id enabled under the built-in provider + const fineTune = "ft:gpt-4o-mini-2024-07-18:acme::BxK9pQ2r" + const settings = makeSettings({ + providers: { + openai: { + apiKey: "sk-test", + enabledModels: [fineTune], + grantSchemaAccess: false, + }, + }, + }) + // Then the ft: prefix is not mistaken for a custom provider + expect(providerForModel(fineTune, settings)).toBe("openai") }) }) @@ -334,6 +365,51 @@ describe("getNextModel", () => { it("returns null when nothing is enabled", () => { expect(getNextModel("gpt-5-mini", {})).toBeNull() }) + + it("stays on the outgoing model's provider when it still has models", () => { + // Given gpt-5.4 was just disabled while OpenAI keeps gpt-5-mini enabled + const previousSettings = makeSettings({ + selectedModel: "gpt-5.4", + providers: { + openai: { + apiKey: "sk-test", + enabledModels: ["gpt-5.4", "gpt-5-mini"], + grantSchemaAccess: false, + }, + anthropic: { + apiKey: "sk-test", + enabledModels: ["claude-opus-5"], + grantSchemaAccess: false, + }, + }, + }) + const updatedSettings = makeSettings({ + selectedModel: "gpt-5.4", + providers: { + openai: { + apiKey: "sk-test", + enabledModels: ["gpt-5-mini"], + grantSchemaAccess: false, + }, + anthropic: { + apiKey: "sk-test", + enabledModels: ["claude-opus-5"], + grantSchemaAccess: false, + }, + }, + }) + + // When picking the next model + const next = getNextModel( + "gpt-5.4", + { openai: ["gpt-5-mini"], anthropic: ["claude-opus-5"] }, + updatedSettings, + previousSettings, + ) + + // Then it falls back within OpenAI instead of hopping to Anthropic + expect(next).toBe("gpt-5-mini") + }) }) describe("getUtilityModel", () => { @@ -495,3 +571,45 @@ describe("getAiPermissions", () => { }) }) }) + +describe("buildListingMetadata", () => { + const GPT5_ERA = Date.UTC(2025, 7, 7) / 1000 + const openaiListing = [ + { id: "gpt-5.4", created: GPT5_ERA + 300 }, + { id: "gpt-5.4-nano", created: GPT5_ERA + 300 }, + { id: "gpt-5-mini", created: GPT5_ERA + 200 }, + { id: "whisper-1", created: GPT5_ERA + 100 }, + ] + + it("derives labels and a cheap utility model from an OpenAI listing", () => { + // Given enabled models including a manually added unlisted id + const metadata = buildListingMetadata("openai", openaiListing, [ + "gpt-5.4", + "my-proxy-model", + ]) + + // Then listed ids get derived labels and unlisted ids keep the raw value + expect(metadata.modelLabels).toEqual({ + "gpt-5.4": "GPT-5.4", + "my-proxy-model": "my-proxy-model", + }) + // And the utility model comes from the cheap tier of the chat pool + expect(metadata.utilityModel).toBe("gpt-5.4-nano") + }) + + it("prefers provider labels and haiku-tier utility for an Anthropic listing", () => { + // Given a listing with provider display names + const listing = [ + { id: "claude-opus-5", label: "Claude Opus 5", created: 300 }, + { id: "claude-haiku-4-5", label: "Claude Haiku 4.5", created: 200 }, + ] + + const metadata = buildListingMetadata("anthropic", listing, [ + "claude-opus-5", + ]) + + // Then the stored label is the provider's and utility picks the haiku tier + expect(metadata.modelLabels).toEqual({ "claude-opus-5": "Claude Opus 5" }) + expect(metadata.utilityModel).toBe("claude-haiku-4-5") + }) +}) diff --git a/src/utils/ai/settings.ts b/src/utils/ai/settings.ts index 19bff7bcb..e2985791d 100644 --- a/src/utils/ai/settings.ts +++ b/src/utils/ai/settings.ts @@ -1,15 +1,14 @@ import type { AiAssistantSettings, CustomProviderDefinition, + ProviderSettings, } from "../../providers/LocalStorageProvider/types" import type { Permissions } from "../tools/permissions" import { getValue } from "../localStorage" import { StoreKey } from "../localStorage/types" import { - computeReasoningModels, filterOpenAiChatModels, formatModelLabel, - matchesListedModel, resolveUtilityModel, UTILITY_MODEL_TIERS, } from "./modelCatalog" @@ -58,18 +57,28 @@ export const makeCustomModelValue = ( export const parseModelValue = ( value: string, + customProviders?: Record, ): { customProviderId: string; rawModel: string } | { rawModel: string } => { const sepIndex = value.indexOf(CUSTOM_MODEL_SEP) if (sepIndex === -1) return { rawModel: value } const candidateProvider = value.slice(0, sepIndex) - // Only treat as namespaced if the prefix is NOT a built-in provider. - if (BUILTIN_PROVIDERS[candidateProvider]) return { rawModel: value } + if (!customProviders || !Object.hasOwn(customProviders, candidateProvider)) { + return { rawModel: value } + } return { customProviderId: candidateProvider, rawModel: value.slice(sepIndex + 1), } } +export const stripModelNamespace = ( + value: string, + providerId: ProviderId, +): string => { + const prefix = `${providerId}${CUSTOM_MODEL_SEP}` + return value.startsWith(prefix) ? value.slice(prefix.length) : value +} + export const getModelLabel = ( modelId: string, providerId: ProviderId, @@ -112,7 +121,7 @@ export const providerForModel = ( settings?: AiAssistantSettings, ): ProviderId | null => { // Check for namespaced custom model value (providerId:modelId) - const parsed = parseModelValue(model) + const parsed = parseModelValue(model, settings?.customProviders) if ("customProviderId" in parsed) return parsed.customProviderId return ( Object.keys(BUILTIN_PROVIDERS).find((providerId) => @@ -166,14 +175,20 @@ export const getNextModel = ( currentModel: string | undefined, enabledModels: Record, settings?: AiAssistantSettings, + previousSettings?: AiAssistantSettings, ): string | null => { const providerOf = (model: string) => { - const parsed = parseModelValue(model) + const parsed = parseModelValue( + model, + settings?.customProviders ?? previousSettings?.customProviders, + ) if ("customProviderId" in parsed) return parsed.customProviderId return ( Object.keys(enabledModels).find((p) => enabledModels[p]?.includes(model), - ) ?? providerForModel(model, settings) + ) ?? + providerForModel(model, settings) ?? + providerForModel(model, previousSettings) ) } const modelProvider = currentModel ? providerOf(currentModel) : null @@ -218,6 +233,35 @@ export const getUtilityModel = ( ) } +export type ProviderSettingsInput = { + apiKey: string + enabledModels: string[] + permissions: Permissions + modelLabels?: Record + utilityModel?: string + reasoningEffort?: "default" | "high" +} + +export const buildProviderSettings = ({ + apiKey, + enabledModels, + permissions, + modelLabels, + utilityModel, + reasoningEffort, +}: ProviderSettingsInput): ProviderSettings => ({ + apiKey, + enabledModels, + grantSchemaAccess: permissions.grantSchemaAccess, + read: permissions.read, + write: permissions.write, + ...(modelLabels && Object.keys(modelLabels).length > 0 + ? { modelLabels } + : {}), + ...(utilityModel ? { utilityModel } : {}), + ...(reasoningEffort === "high" ? { reasoningEffort: "high" as const } : {}), +}) + /** * Derives the listing-dependent provider settings captured at Save time: * labels for enabled models, the utility model, and the reasoning gate. @@ -225,7 +269,6 @@ export const getUtilityModel = ( export type ListingMetadata = { modelLabels: Record utilityModel?: string - reasoningModels?: string[] } export const buildListingMetadata = ( @@ -241,13 +284,12 @@ export const buildListingMetadata = ( ) const modelLabels: Record = {} for (const id of enabledModels) { - const label = listing.find((m) => matchesListedModel(id, m.id))?.label - if (label) modelLabels[id] = label + const listed = listing.find((m) => m.id === id) + modelLabels[id] = listed ? (listed.label ?? formatModelLabel(id)) : id } return { modelLabels, ...(utilityModel ? { utilityModel } : {}), - ...(isOpenAi ? { reasoningModels: computeReasoningModels(listing) } : {}), } } From 7ba8104884bce05d2bd66897fd09d5be4ee68dca Mon Sep 17 00:00:00 2001 From: emrberk Date: Fri, 4 Sep 2026 17:09:10 +0300 Subject: [PATCH 5/9] fix(ai): preserve custom models and settings drafts --- e2e/tests/console/aiProviderSetup.spec.js | 23 +++++++++ .../SetupAIAssistant/SettingsModal.tsx | 23 +++++---- src/utils/ai/settings.test.ts | 47 +++++++++++++++++++ src/utils/ai/settings.ts | 25 +++++++--- 4 files changed, 101 insertions(+), 17 deletions(-) diff --git a/e2e/tests/console/aiProviderSetup.spec.js b/e2e/tests/console/aiProviderSetup.spec.js index 43f578620..ae70d6739 100644 --- a/e2e/tests/console/aiProviderSetup.spec.js +++ b/e2e/tests/console/aiProviderSetup.spec.js @@ -397,6 +397,10 @@ describe("ai provider setup flows", () => { interceptOpenAIListing() cy.getByDataHook("ai-assistant-settings-button").click() cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("reasoning-trigger").click() + cy.getByDataHook("reasoning-level-high").click() + cy.getByDataHook("permissions-trigger").click() + cy.getByDataHook("permission-level-write").click() cy.getByDataHook("ai-settings-manage-models").click() cy.wait("@openaiListing") @@ -411,9 +415,28 @@ describe("ai provider setup flows", () => { cy.window().then((win) => { const settings = readAiSettings(win) expect(settings.selectedModel).to.equal("gpt-5-mini") + expect(settings.providers.openai.grantSchemaAccess).to.equal(true) + expect(settings.providers.openai.read).to.equal(false) + expect(settings.providers.openai.write).to.equal(false) + expect(settings.providers.openai.reasoningEffort).to.equal(undefined) + }) + + // Then cancelling the parent discards its permission and reasoning drafts + cy.getByDataHook("ai-settings-cancel").click() + cy.reload() + cy.getEditor().should("be.visible") + cy.window().then((win) => { + const settings = readAiSettings(win) + expect(settings.providers.openai.grantSchemaAccess).to.equal(true) + expect(settings.providers.openai.read).to.equal(false) + expect(settings.providers.openai.write).to.equal(false) + expect(settings.providers.openai.reasoningEffort).to.equal(undefined) }) // When the API key changes to a different one and validates + interceptOpenAIListing() + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-settings-provider-openai").click() cy.getByDataHook("ai-settings-edit-api-key").click() cy.getByDataHook("ai-settings-api-key").should("not.have.attr", "readonly") cy.getByDataHook("ai-settings-api-key").clear().type("key-two") diff --git a/src/components/SetupAIAssistant/SettingsModal.tsx b/src/components/SetupAIAssistant/SettingsModal.tsx index afa7e589e..9950c331e 100644 --- a/src/components/SetupAIAssistant/SettingsModal.tsx +++ b/src/components/SetupAIAssistant/SettingsModal.tsx @@ -984,7 +984,16 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { [providerId]: result.utilityModel, })) - const perms = permissions[providerId] + const storedProvider = aiAssistantSettings.providers?.[providerId] + // The nested dialog commits its API key and model preferences only. + // Permission and reasoning edits remain drafts until Save Settings. + const persistedPermissions: Permissions = storedProvider + ? { + grantSchemaAccess: storedProvider.grantSchemaAccess, + read: storedProvider.read === true, + write: storedProvider.write === true, + } + : { grantSchemaAccess: true, read: false, write: false } const updatedSettings: AiAssistantSettings = { ...aiAssistantSettings, providers: { @@ -992,10 +1001,10 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { [providerId]: buildProviderSettings({ apiKey: apiKeys[providerId] ?? "", enabledModels: result.enabledModels, - permissions: perms, + permissions: persistedPermissions, modelLabels: result.modelLabels, utilityModel: result.utilityModel, - reasoningEffort: reasoningEffort[providerId], + reasoningEffort: storedProvider?.reasoningEffort ?? "default", }), }, } @@ -1018,13 +1027,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { updateSettings(StoreKey.AI_ASSISTANT_SETTINGS, updatedSettings) toast.success("Model preferences updated") }, - [ - aiAssistantSettings, - apiKeys, - permissions, - reasoningEffort, - updateSettings, - ], + [aiAssistantSettings, apiKeys, updateSettings], ) const currentProviderValidated = validatedApiKeys[selectedProvider] diff --git a/src/utils/ai/settings.test.ts b/src/utils/ai/settings.test.ts index d7cc86b2e..4dcda54f6 100644 --- a/src/utils/ai/settings.test.ts +++ b/src/utils/ai/settings.test.ts @@ -61,6 +61,22 @@ describe("reconcileSettings", () => { ]) }) + it("also collapses legacy reasoning variants for built-in Anthropic models", () => { + const settings = makeSettings({ + selectedModel: "claude-sonnet@reasoning=high", + providers: { + anthropic: { + apiKey: "sk-test", + enabledModels: ["claude-sonnet@reasoning=high"], + grantSchemaAccess: false, + }, + }, + }) + const result = reconcileSettings(settings) + expect(result.providers.anthropic!.enabledModels).toEqual(["claude-sonnet"]) + expect(result.selectedModel).toBe("claude-sonnet") + }) + it("folds a selected high variant into reasoningEffort", () => { // Given a user who ran the high variant const settings = makeSettings({ @@ -121,6 +137,37 @@ describe("reconcileSettings", () => { ]) }) + it("preserves reasoning-like suffixes in custom provider model ids", () => { + const customModels = [ + "vendor-model@reasoning=high", + "vendor-model@reasoning=medium", + "vendor-model@reasoning=low", + ] + const enabledModels = customModels.map((model) => `custom-1:${model}`) + const settings = makeSettings({ + selectedModel: enabledModels[0], + customProviders: { + "custom-1": { + type: "openai-chat-completions", + name: "Test", + baseURL: "http://localhost:11434/v1", + contextWindow: 100_000, + models: customModels, + }, + }, + providers: { + "custom-1": { + apiKey: "", + enabledModels, + grantSchemaAccess: false, + }, + }, + }) + const result = reconcileSettings(settings) + expect(result.providers["custom-1"]!.enabledModels).toEqual(enabledModels) + expect(result.selectedModel).toBe(enabledModels[0]) + }) + it("is idempotent", () => { const settings = makeSettings({ selectedModel: "gpt-5.4@reasoning=high", diff --git a/src/utils/ai/settings.ts b/src/utils/ai/settings.ts index e2985791d..7090ef4f9 100644 --- a/src/utils/ai/settings.ts +++ b/src/utils/ai/settings.ts @@ -334,13 +334,12 @@ export const reconcileSettings = ( const providerSettings = result.providers[providerKey] if (!providerSettings?.enabledModels) continue + const isBuiltinProvider = Object.hasOwn(BUILTIN_PROVIDERS, providerKey) const selectedHighVariant = + isBuiltinProvider && selectedModel !== undefined && selectedModel.endsWith("@reasoning=high") && providerSettings.enabledModels.includes(selectedModel) - const collapsed = [ - ...new Set(providerSettings.enabledModels.map(collapseLegacyVariant)), - ] const validCustomIds = settings.customProviders?.[providerKey] ? new Set( settings.customProviders[providerKey].models.map((m) => @@ -350,15 +349,27 @@ export const reconcileSettings = ( : null result.providers[providerKey] = { ...providerSettings, - enabledModels: BUILTIN_PROVIDERS[providerKey] - ? collapsed - : collapsed.filter((id) => validCustomIds?.has(id)), + enabledModels: isBuiltinProvider + ? [ + ...new Set( + providerSettings.enabledModels.map(collapseLegacyVariant), + ), + ] + : providerSettings.enabledModels.filter((id) => + validCustomIds?.has(id), + ), ...(selectedHighVariant ? { reasoningEffort: "high" as const } : {}), } } if (result.selectedModel !== undefined) { - result.selectedModel = collapseLegacyVariant(result.selectedModel) + const selectedProvider = providerForModel(result.selectedModel, settings) + if ( + selectedProvider !== null && + Object.hasOwn(BUILTIN_PROVIDERS, selectedProvider) + ) { + result.selectedModel = collapseLegacyVariant(result.selectedModel) + } } result.selectedModel = getSelectedModel(result) ?? undefined From 03df4ef0097c1ea3ed33c99bd6071844b1007103 Mon Sep 17 00:00:00 2001 From: emrberk Date: Fri, 4 Sep 2026 19:13:12 +0300 Subject: [PATCH 6/9] model label fixes, handle overload --- e2e/questdb | 2 +- e2e/tests/console/aiAssistant.spec.js | 2 +- e2e/tests/console/aiProviderSetup.spec.js | 73 ++++++++++++++++--- .../SetupAIAssistant/ConfigurationModal.tsx | 7 -- .../SetupAIAssistant/ManageModelsModal.tsx | 50 ++++++++----- .../SetupAIAssistant/ModelPicker.tsx | 31 +------- src/utils/ai/anthropicProvider.test.ts | 27 ++++++- src/utils/ai/anthropicProvider.ts | 3 +- src/utils/ai/modelCatalog.test.ts | 40 +++++++--- src/utils/ai/modelCatalog.ts | 24 +++--- src/utils/ai/openaiShared.test.ts | 23 +++++- src/utils/ai/openaiShared.ts | 3 +- src/utils/ai/settings.test.ts | 4 +- 13 files changed, 194 insertions(+), 95 deletions(-) diff --git a/e2e/questdb b/e2e/questdb index 090a22720..a88326220 160000 --- a/e2e/questdb +++ b/e2e/questdb @@ -1 +1 @@ -Subproject commit 090a2272082337f5f986b80e348f9e9d659ab08f +Subproject commit a883262205d9c03ec29f69b6cd402baf6a6b9089 diff --git a/e2e/tests/console/aiAssistant.spec.js b/e2e/tests/console/aiAssistant.spec.js index 39e967ecf..5db93f04f 100644 --- a/e2e/tests/console/aiAssistant.spec.js +++ b/e2e/tests/console/aiAssistant.spec.js @@ -3384,7 +3384,7 @@ describe("custom providers", () => { cy.getByDataHook("ai-settings-model-dropdown").click() cy.getByDataHook("ai-settings-model-item-label") - .contains("GPT-5 Mini") + .contains("GPT 5 Mini") .click() cy.getByDataHook("chat-window-new").click() diff --git a/e2e/tests/console/aiProviderSetup.spec.js b/e2e/tests/console/aiProviderSetup.spec.js index ae70d6739..cd0066ade 100644 --- a/e2e/tests/console/aiProviderSetup.spec.js +++ b/e2e/tests/console/aiProviderSetup.spec.js @@ -260,7 +260,7 @@ describe("ai provider setup flows", () => { expect(settings.providers.openai.reasoningEffort).to.equal("high") expect(settings.providers.openai.utilityModel).to.equal("gpt-5-nano") expect(settings.providers.openai.modelLabels).to.deep.equal({ - "gpt-5.4": "GPT-5.4", + "gpt-5.4": "GPT 5.4", "my-proxy-model": "my-proxy-model", }) }) @@ -353,7 +353,7 @@ describe("ai provider setup flows", () => { cy.wait("@openaiListing") // Then the picker auto-opens from the validation fetch, with no extra request - cy.getByDataHook("manage-models-model-row").should("have.length", 4) + cy.getByDataHook("manage-models-model-row").should("have.length", 5) cy.get("@openaiListing.all").should("have.length", 1) // When the picker is cancelled with nothing selected @@ -366,19 +366,19 @@ describe("ai provider setup flows", () => { // When validation runs again and Select All is used cy.getByDataHook("ai-settings-test-api").click() cy.wait("@openaiListing") - cy.getByDataHook("manage-models-model-row").should("have.length", 4) - cy.getByDataHook("manage-models-show-all").click() + cy.getByDataHook("manage-models-model-row").should("have.length", 5) cy.getByDataHook("manage-models-select-all").click() cy.getByDataHook("manage-models-save").click() cy.getByDataHook("manage-models-save").should("not.exist") - // Then only the curated chat models persist — hidden ids stay out + // Then every listed chat model, including dated snapshots, persists cy.window().then((win) => { const settings = readAiSettings(win) expect(settings.providers.openai.enabledModels).to.deep.equal([ "gpt-5.4", "gpt-5-mini", "gpt-5", + "gpt-5-2025-08-06", "gpt-5-nano", ]) }) @@ -390,8 +390,8 @@ describe("ai provider setup flows", () => { // Then the picks survive and an OpenAI model can be selected cy.getByDataHook("ai-settings-model-dropdown").click() - cy.getByDataHook("ai-settings-model-item").should("have.length", 6) - cy.getByDataHook("ai-settings-model-item").contains("GPT-5.4").click() + cy.getByDataHook("ai-settings-model-item").should("have.length", 7) + cy.getByDataHook("ai-settings-model-item").contains("GPT 5.4").click() // When the picker reopens manually it refetches a fresh listing interceptOpenAIListing() @@ -444,7 +444,7 @@ describe("ai provider setup flows", () => { cy.wait("@openaiListing") // Then the old key's picks are cleared in the picker - cy.getByDataHook("manage-models-model-row").should("have.length", 4) + cy.getByDataHook("manage-models-model-row").should("have.length", 5) cy.getByDataHook("manage-models-model-row") .find("input[type=checkbox]:checked") .should("have.length", 0) @@ -494,8 +494,7 @@ describe("ai provider setup flows", () => { cy.getByDataHook("manage-models-model-chip").contains("gpt-5.4-2026-03-05") cy.getByDataHook("manage-models-model-chip").contains("my-proxy-model") - // When all models are revealed, alias and dated rows toggle independently - cy.getByDataHook("manage-models-show-all").click() + // The plain alias and dated snapshot rows toggle independently cy.getByDataHook("manage-models-model-row") .contains(/^gpt-5$/) .closest("label") @@ -529,13 +528,63 @@ describe("ai provider setup flows", () => { // And the dropdown shows derived labels but never invents one for manual ids cy.getByDataHook("ai-settings-cancel").click() cy.getByDataHook("ai-settings-model-dropdown").click() - cy.getByDataHook("ai-settings-model-item").contains("GPT-5.4") + cy.getByDataHook("ai-settings-model-item").contains("GPT 5.4") cy.getByDataHook("ai-settings-model-item").contains("my-proxy-model") cy.getByDataHook("ai-settings-model-item") .contains("My-proxy-model") .should("not.exist") }) + it("blocks model changes and identifies a rate-limited listing", () => { + // Given an existing OpenAI configuration whose model listing is rate limited + cy.loadConsoleWithAuth(false, { + "ai.assistant.settings": JSON.stringify({ + selectedModel: "gpt-5.4", + providers: { + openai: { + apiKey: "test-openai-key", + enabledModels: ["gpt-5.4"], + grantSchemaAccess: false, + }, + }, + }), + }) + cy.intercept("GET", OPENAI_MODELS_URL, { + statusCode: 429, + headers: { "retry-after": "0" }, + body: { + error: { + message: "Rate limit reached", + type: "rate_limit_error", + param: null, + code: "rate_limit_exceeded", + }, + }, + }).as("openaiListing") + + // When Manage Models attempts to refresh the provider listing + cy.getByDataHook("ai-assistant-settings-button").click() + cy.getByDataHook("ai-settings-provider-openai").click() + cy.getByDataHook("ai-settings-manage-models").click() + cy.wait("@openaiListing") + + // Then the failure is identified and no model changes can be made + cy.getByDataHook("manage-models-fetch-error").should( + "contain", + "rate or usage limit was reached", + ) + cy.getByDataHook("manage-models-model-row").should("not.exist") + cy.getByDataHook("manage-models-manual-model-input").should("not.exist") + cy.getByDataHook("manage-models-save").should("be.disabled") + + // And the saved configuration is untouched + cy.window().then((win) => { + expect(readAiSettings(win).providers.openai.enabledModels).to.deep.equal([ + "gpt-5.4", + ]) + }) + }) + it("survives interruptions: wizard escape, tab switch mid-validation, stale key edits", () => { // Given a fresh console and a slow listing response cy.loadConsoleWithAuth() @@ -884,7 +933,7 @@ describe("ai provider setup flows", () => { }) it("should work with multiple providers", () => { - const openaiEnabledModels = ["GPT-5.4", "GPT-5 Mini"] + const openaiEnabledModels = ["GPT 5.4", "GPT 5 Mini"] const anthropicEnabledModels = ["Claude Opus 4.5", "Claude Sonnet 4.5"] // Given - Set up OpenAI provider first diff --git a/src/components/SetupAIAssistant/ConfigurationModal.tsx b/src/components/SetupAIAssistant/ConfigurationModal.tsx index 6301e7c94..a13edb8a0 100644 --- a/src/components/SetupAIAssistant/ConfigurationModal.tsx +++ b/src/components/SetupAIAssistant/ConfigurationModal.tsx @@ -410,12 +410,6 @@ const StepTwoContent = ({ ? filterOpenAiChatModels(listing) : sortModelsNewestFirst(listing) : [] - const hiddenModels = - listing && isOpenAi - ? sortModelsNewestFirst( - listing.filter((m) => !pickerModels.some((p) => p.id === m.id)), - ) - : undefined return ( @@ -462,7 +456,6 @@ const StepTwoContent = ({ void + onFetchFailedChange: (failed: boolean) => void } const BuiltinModelsContent = forwardRef< @@ -134,13 +135,20 @@ const BuiltinModelsContent = forwardRef< BuiltinModelsContentProps >( ( - { providerId, apiKey, enabledModels, initialListing, onLoadingChange }, + { + providerId, + apiKey, + enabledModels, + initialListing, + onLoadingChange, + onFetchFailedChange, + }, ref, ) => { const [listing, setListing] = useState( initialListing ?? null, ) - const [fetchFailed, setFetchFailed] = useState(false) + const [fetchError, setFetchError] = useState(null) const [selectedModels, setSelectedModels] = useState( initialListing ? enabledModels : [], ) @@ -153,12 +161,6 @@ const BuiltinModelsContent = forwardRef< ? filterOpenAiChatModels(listing) : sortModelsNewestFirst(listing) : [] - const hiddenModels = - listing && isOpenAi - ? sortModelsNewestFirst( - listing.filter((m) => !pickerModels.some((p) => p.id === m.id)), - ) - : undefined const selectionWithPending = () => { const pending = manualInput.trim() @@ -193,23 +195,31 @@ const BuiltinModelsContent = forwardRef< const doFetch = async () => { if (initialListing) { + onFetchFailedChange(false) onLoadingChange(false) return } + onFetchFailedChange(false) onLoadingChange(true) + const provider = createProviderByType( + BUILTIN_PROVIDERS[providerId].type, + providerId, + apiKey, + ) try { - const provider = createProviderByType( - BUILTIN_PROVIDERS[providerId].type, - providerId, - apiKey, - ) const models = await provider.listModels() if (cancelled) return setListing(models) setSelectedModels(enabledModels) - } catch { + } catch (error) { if (cancelled) return - setFetchFailed(true) + const classified = provider.classifyError(error, () => {}) + setFetchError( + classified.type === "rate_limit" + ? classified.message + : "Could not fetch models from the provider. Check your API key and connection, then try again.", + ) + onFetchFailedChange(true) } finally { if (!cancelled) { setIsLoading(false) @@ -234,12 +244,11 @@ const BuiltinModelsContent = forwardRef< ) } - if (fetchFailed) { + if (fetchError) { return ( - Could not fetch models from the provider. Check your API key and - connection, then try again. + {fetchError} ) @@ -249,7 +258,6 @@ const BuiltinModelsContent = forwardRef< { const { open, onOpenChange, providerId } = props const [error, setError] = useState(null) const [modelsLoading, setModelsLoading] = useState(true) + const [modelsFetchFailed, setModelsFetchFailed] = useState(false) const modelSettingsRef = useRef(null) const builtinModelsRef = useRef(null) @@ -368,6 +377,7 @@ export const ManageModelsModal = (props: ManageModelsModalProps) => { enabledModels={props.enabledModels} initialListing={props.initialListing} onLoadingChange={setModelsLoading} + onFetchFailedChange={setModelsFetchFailed} /> )} @@ -390,7 +400,7 @@ export const ManageModelsModal = (props: ManageModelsModalProps) => { variant="primary" data-hook="manage-models-save" onClick={handleSave} - disabled={modelsLoading} + disabled={modelsLoading || modelsFetchFailed} > Save diff --git a/src/components/SetupAIAssistant/ModelPicker.tsx b/src/components/SetupAIAssistant/ModelPicker.tsx index e36ff05c5..28d02d129 100644 --- a/src/components/SetupAIAssistant/ModelPicker.tsx +++ b/src/components/SetupAIAssistant/ModelPicker.tsx @@ -1,4 +1,4 @@ -import React, { useState } from "react" +import React from "react" import styled from "styled-components" import { XIcon } from "@phosphor-icons/react" import { Box } from "../Box" @@ -9,7 +9,6 @@ import { Input } from "../Input" import { Text } from "../Text" import { TextButton } from "../TextButton" import type { ProviderModel } from "../../utils/ai" -import { sortModelsNewestFirst } from "../../utils/ai" const PickerSection = styled(Box).attrs({ flexDirection: "column", @@ -74,11 +73,6 @@ const ModelIdText = styled(Text)` color: ${({ theme }) => theme.color.contentSecondary}; ` -const ShowAllButton = styled(TextButton)` - font-size: 1.3rem; - align-self: flex-start; -` - const HelperText = styled(Text)` font-size: 1.3rem; font-weight: 400; @@ -129,7 +123,6 @@ const ChipRemoveButton = styled(IconButton)` export type ModelPickerProps = { listedModels: ProviderModel[] - hiddenModels?: ProviderModel[] selectedModels: string[] manualInput: string dataHookPrefix: string @@ -140,7 +133,6 @@ export type ModelPickerProps = { export const ModelPicker = ({ listedModels, - hiddenModels, selectedModels, manualInput, dataHookPrefix, @@ -148,15 +140,9 @@ export const ModelPicker = ({ onSelectionChange, onManualInputChange, }: ModelPickerProps) => { - const [showAll, setShowAll] = useState(false) - - const visibleModels = - showAll && hiddenModels?.length - ? sortModelsNewestFirst([...listedModels, ...hiddenModels]) - : listedModels const isRowChecked = (rowId: string) => selectedModels.includes(rowId) const manualModels = selectedModels.filter( - (selected) => !visibleModels.some((m) => m.id === selected), + (selected) => !listedModels.some((m) => m.id === selected), ) const handleToggleRow = (rowId: string) => { @@ -176,7 +162,7 @@ export const ModelPicker = ({ const handleDeselectAll = () => { onSelectionChange( - selectedModels.filter((s) => !visibleModels.some((m) => m.id === s)), + selectedModels.filter((s) => !listedModels.some((m) => m.id === s)), ) } @@ -216,7 +202,7 @@ export const ModelPicker = ({ - {visibleModels.map((model) => { + {listedModels.map((model) => { const label = labelFor ? labelFor(model) : model.id return ( - {!showAll && !!hiddenModels?.length && ( - setShowAll(true)} - > - Show all models - - )} Don't see your model? Add it manually: diff --git a/src/utils/ai/anthropicProvider.test.ts b/src/utils/ai/anthropicProvider.test.ts index b6f0a523b..7bba9c0fc 100644 --- a/src/utils/ai/anthropicProvider.test.ts +++ b/src/utils/ai/anthropicProvider.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest" -import { toNativeMessages } from "./anthropicProvider" +import Anthropic from "@anthropic-ai/sdk" +import { createAnthropicProvider, toNativeMessages } from "./anthropicProvider" import type { Message, ToolCall } from "./types" const toolCall = (over: Partial = {}): ToolCall => ({ @@ -10,6 +11,30 @@ const toolCall = (over: Partial = {}): ToolCall => ({ ...over, }) +describe("anthropicProvider.classifyError", () => { + it("reports a 429 as a rate or usage limit", () => { + const provider = createAnthropicProvider("test-key") + const error = new Anthropic.RateLimitError( + 429, + { + type: "error", + error: { + type: "rate_limit_error", + message: "Rate limit reached", + }, + }, + "Rate limit reached", + new Headers(), + ) + + expect(provider.classifyError(error, () => {})).toMatchObject({ + type: "rate_limit", + message: + "The provider's rate or usage limit was reached. Check your provider account limits or try again later.", + }) + }) +}) + describe("anthropicProvider.toNativeMessages", () => { it("converts a plain user message to a user role param", () => { // Given a single user message diff --git a/src/utils/ai/anthropicProvider.ts b/src/utils/ai/anthropicProvider.ts index cd2e139dc..9ca7e4d61 100644 --- a/src/utils/ai/anthropicProvider.ts +++ b/src/utils/ai/anthropicProvider.ts @@ -717,7 +717,8 @@ export function createAnthropicProvider( if (error instanceof Anthropic.RateLimitError) { return { type: "rate_limit", - message: "Rate limit exceeded. Please try again later.", + message: + "The provider's rate or usage limit was reached. Check your provider account limits or try again later.", details: error.message, } } diff --git a/src/utils/ai/modelCatalog.test.ts b/src/utils/ai/modelCatalog.test.ts index c85b00ec8..41bbea2d5 100644 --- a/src/utils/ai/modelCatalog.test.ts +++ b/src/utils/ai/modelCatalog.test.ts @@ -30,25 +30,43 @@ describe("stripDateSuffix", () => { describe("formatModelLabel", () => { it("derives labels from OpenAI ids", () => { - expect(formatModelLabel("gpt-5-mini")).toBe("GPT-5 Mini") - expect(formatModelLabel("gpt-5.4")).toBe("GPT-5.4") - expect(formatModelLabel("gpt-5.6-luna")).toBe("GPT-5.6 Luna") - expect(formatModelLabel("gpt-4o")).toBe("GPT-4o") + expect(formatModelLabel("gpt-5-mini")).toBe("GPT 5 Mini") + expect(formatModelLabel("gpt-5.4")).toBe("GPT 5.4") + expect(formatModelLabel("gpt-5.6-luna")).toBe("GPT 5.6 Luna") + expect(formatModelLabel("gpt-4o")).toBe("GPT 4o") expect(formatModelLabel("o4-mini")).toBe("o4 Mini") }) - it("joins consecutive version numbers with dots", () => { + it("formats version numbers without provider-specific separators", () => { + expect(formatModelLabel("gpt-6")).toBe("GPT 6") + expect(formatModelLabel("gpt-6-7")).toBe("GPT 6.7") + expect(formatModelLabel("gpt-5.6-7")).toBe("GPT 5.6.7") + expect(formatModelLabel("gpt-5.6-something")).toBe("GPT 5.6 Something") + expect(formatModelLabel("something-1.2")).toBe("Something 1.2") expect(formatModelLabel("claude-sonnet-4-5")).toBe("Claude Sonnet 4.5") + expect(formatModelLabel("claude-sonnet-5")).toBe("Claude Sonnet 5") + expect(formatModelLabel("claude-sonnet-6-7")).toBe("Claude Sonnet 6.7") + expect(formatModelLabel("claude-sonnet-6.7-something")).toBe( + "Claude Sonnet 6.7 Something", + ) + expect(formatModelLabel("claude-sonnet-6-7-something")).toBe( + "Claude Sonnet 6.7 Something", + ) }) - it("drops the date suffix", () => { - expect(formatModelLabel("claude-opus-4-5-20251101")).toBe("Claude Opus 4.5") - expect(formatModelLabel("gpt-5.4-nano-2026-03-17")).toBe("GPT-5.4 Nano") + it("preserves date suffixes", () => { + expect(formatModelLabel("gpt-5.4-20250815")).toBe("GPT 5.4 (20250815)") + expect(formatModelLabel("claude-opus-4-5-20251101")).toBe( + "Claude Opus 4.5 (20251101)", + ) + expect(formatModelLabel("gpt-5.4-nano-2026-03-17")).toBe( + "GPT 5.4 Nano (2026-03-17)", + ) }) }) describe("filterOpenAiChatModels", () => { - it("drops known non-chat models and dated snapshots", () => { + it("drops known non-chat models but keeps dated snapshots", () => { // Given a listing with chat models, noise, and dated snapshots const listing = [ model("gpt-5.4", AUG_2025 + 300), @@ -64,8 +82,8 @@ describe("filterOpenAiChatModels", () => { ] // When the filter runs const kept = filterOpenAiChatModels(listing).map((m) => m.id) - // Then only the plain chat model remains - expect(kept).toEqual(["gpt-5.4"]) + // Then both the plain chat model and its dated snapshot remain + expect(kept).toEqual(["gpt-5.4", "gpt-5.4-2026-03-05"]) }) it("hides generations older than gpt-5 by default", () => { diff --git a/src/utils/ai/modelCatalog.ts b/src/utils/ai/modelCatalog.ts index 749f051c6..2a30fb4e8 100644 --- a/src/utils/ai/modelCatalog.ts +++ b/src/utils/ai/modelCatalog.ts @@ -6,8 +6,8 @@ export type ProviderModel = { const DATE_SUFFIX = /-(\d{8}|\d{4}-\d{2}-\d{2}|\d{4})$/ -// Everything since the gpt-5 launch reasons, and belongs in the default -// picker view; older generations stay reachable via "Show all models". +// Models released since GPT-5 are eligible for the picker; older models remain +// available through manual entry. const GPT5_LAUNCH_START = Date.UTC(2025, 7, 1) / 1000 const OPENAI_NON_CHAT_TOKENS = [ @@ -41,7 +41,9 @@ export const stripDateSuffix = (id: string): string => const isNumericToken = (token: string): boolean => /^[\d.]+$/.test(token) export const formatModelLabel = (id: string): string => { - const tokens = stripDateSuffix(id).split("-") + const dateSuffix = id.match(DATE_SUFFIX) + const baseId = dateSuffix ? id.slice(0, -dateSuffix[0].length) : id + const tokens = baseId.split("-") const parts: string[] = [] for (const token of tokens) { if (token.toLowerCase() === "gpt") { @@ -49,19 +51,22 @@ export const formatModelLabel = (id: string): string => { continue } const previous = parts[parts.length - 1] - if (/\d/.test(token)) { - if (previous === "GPT") { - parts[parts.length - 1] = `GPT-${token}` - } else if (previous && isNumericToken(previous)) { + if (isNumericToken(token)) { + if (previous && isNumericToken(previous)) { parts[parts.length - 1] = `${previous}.${token}` } else { parts.push(token) } continue } + if (/\d/.test(token)) { + parts.push(token) + continue + } parts.push(token.charAt(0).toUpperCase() + token.slice(1)) } - return parts.join(" ") || id + const label = parts.join(" ") || id + return dateSuffix ? `${label} (${dateSuffix[1]})` : label } export const sortModelsNewestFirst = ( @@ -72,8 +77,7 @@ export const sortModelsNewestFirst = ( ) const isOpenAiNonChatModel = (id: string): boolean => - OPENAI_NON_CHAT_TOKENS.some((token) => id.includes(token)) || - DATE_SUFFIX.test(id) + OPENAI_NON_CHAT_TOKENS.some((token) => id.toLowerCase().includes(token)) export const filterOpenAiChatModels = ( models: ProviderModel[], diff --git a/src/utils/ai/openaiShared.test.ts b/src/utils/ai/openaiShared.test.ts index 7afaf06d5..6ad765b4c 100644 --- a/src/utils/ai/openaiShared.test.ts +++ b/src/utils/ai/openaiShared.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest" import OpenAI from "openai" -import { isReasoningRejection } from "./openaiShared" +import { classifyOpenAIError, isReasoningRejection } from "./openaiShared" const apiError = (status: number, param: string | null, message: string) => new OpenAI.APIError( @@ -48,3 +48,24 @@ describe("isReasoningRejection", () => { expect(isReasoningRejection(new Error("reasoning failed"))).toBe(false) }) }) + +describe("classifyOpenAIError", () => { + it("reports a 429 as a rate or usage limit", () => { + const error = new OpenAI.RateLimitError( + 429, + { + message: "Rate limit reached", + type: "rate_limit_error", + code: "rate_limit_exceeded", + }, + "Rate limit reached", + new Headers(), + ) + + expect(classifyOpenAIError(error, () => {})).toMatchObject({ + type: "rate_limit", + message: + "The provider's rate or usage limit was reached. Check your provider account limits or try again later.", + }) + }) +}) diff --git a/src/utils/ai/openaiShared.ts b/src/utils/ai/openaiShared.ts index d8158b512..7d5a9e8f8 100644 --- a/src/utils/ai/openaiShared.ts +++ b/src/utils/ai/openaiShared.ts @@ -93,7 +93,8 @@ export function classifyOpenAIError( if (error instanceof OpenAI.RateLimitError) { return { type: "rate_limit", - message: "Rate limit exceeded. Please try again later.", + message: + "The provider's rate or usage limit was reached. Check your provider account limits or try again later.", details: error.message, } } diff --git a/src/utils/ai/settings.test.ts b/src/utils/ai/settings.test.ts index 4dcda54f6..ffe97fbd5 100644 --- a/src/utils/ai/settings.test.ts +++ b/src/utils/ai/settings.test.ts @@ -315,7 +315,7 @@ describe("getAllModelOptions", () => { // Then the stored label wins and the formatter fills the gap expect(options).toEqual([ { label: "GPT-5.4 (Custom)", value: "gpt-5.4", provider: "openai" }, - { label: "GPT-5 Mini", value: "gpt-5-mini", provider: "openai" }, + { label: "GPT 5 Mini", value: "gpt-5-mini", provider: "openai" }, ]) }) @@ -637,7 +637,7 @@ describe("buildListingMetadata", () => { // Then listed ids get derived labels and unlisted ids keep the raw value expect(metadata.modelLabels).toEqual({ - "gpt-5.4": "GPT-5.4", + "gpt-5.4": "GPT 5.4", "my-proxy-model": "my-proxy-model", }) // And the utility model comes from the cheap tier of the chat pool From 803cbe3f4f685d46f4df32577796bddefa8ac25e Mon Sep 17 00:00:00 2001 From: emrberk Date: Sun, 6 Sep 2026 16:08:42 +0300 Subject: [PATCH 7/9] fix CI --- e2e/tests/console/aiProviderSetup.spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/tests/console/aiProviderSetup.spec.js b/e2e/tests/console/aiProviderSetup.spec.js index cd0066ade..50f927ac6 100644 --- a/e2e/tests/console/aiProviderSetup.spec.js +++ b/e2e/tests/console/aiProviderSetup.spec.js @@ -457,6 +457,7 @@ describe("ai provider setup flows", () => { expect(settings.providers.openai.enabledModels).to.deep.equal([ "gpt-5-mini", "gpt-5", + "gpt-5-2025-08-06", "gpt-5-nano", ]) }) From be486da72a35590cfbeafbc0a3588d0ab1e705ed Mon Sep 17 00:00:00 2001 From: emrberk Date: Sun, 6 Sep 2026 19:01:26 +0300 Subject: [PATCH 8/9] better error message on validation, model namespace migration --- e2e/tests/console/aiProviderSetup.spec.js | 91 ++++- e2e/utils/aiAssistant.js | 19 +- .../SetupAIAssistant/ConfigurationModal.tsx | 11 +- .../SetupAIAssistant/SettingsModal.tsx | 48 +-- src/providers/LocalStorageProvider/index.tsx | 26 +- src/providers/LocalStorageProvider/types.ts | 4 + .../ai/anthropicProvider.namespace.test.ts | 85 ++++ src/utils/ai/index.ts | 3 +- src/utils/ai/modelListingError.test.ts | 43 +++ src/utils/ai/modelListingError.ts | 36 ++ src/utils/ai/openaiProvider.fallback.test.ts | 4 +- src/utils/ai/settings.test.ts | 364 +++++++++++++----- src/utils/ai/settings.ts | 155 ++------ src/utils/localStorage/migrate.ts | 163 ++++++++ 14 files changed, 783 insertions(+), 269 deletions(-) create mode 100644 src/utils/ai/anthropicProvider.namespace.test.ts create mode 100644 src/utils/ai/modelListingError.test.ts create mode 100644 src/utils/ai/modelListingError.ts create mode 100644 src/utils/localStorage/migrate.ts diff --git a/e2e/tests/console/aiProviderSetup.spec.js b/e2e/tests/console/aiProviderSetup.spec.js index 50f927ac6..57ae16e52 100644 --- a/e2e/tests/console/aiProviderSetup.spec.js +++ b/e2e/tests/console/aiProviderSetup.spec.js @@ -252,13 +252,15 @@ describe("ai provider setup flows", () => { // Then the persisted settings carry the whole configuration cy.window().then((win) => { const settings = readAiSettings(win) - expect(settings.selectedModel).to.equal("gpt-5.4") + expect(settings.selectedModel).to.equal("openai:gpt-5.4") expect(settings.providers.openai.enabledModels).to.deep.equal([ - "gpt-5.4", - "my-proxy-model", + "openai:gpt-5.4", + "openai:my-proxy-model", ]) expect(settings.providers.openai.reasoningEffort).to.equal("high") - expect(settings.providers.openai.utilityModel).to.equal("gpt-5-nano") + expect(settings.providers.openai.utilityModel).to.equal( + "openai:gpt-5-nano", + ) expect(settings.providers.openai.modelLabels).to.deep.equal({ "gpt-5.4": "GPT 5.4", "my-proxy-model": "my-proxy-model", @@ -268,9 +270,11 @@ describe("ai provider setup flows", () => { // When a chat message is sent cy.intercept("POST", PROVIDERS.openai.endpoint, (req) => { if (isTitleRequest("openai", req.body)) { + expect(req.body.model).to.equal("gpt-5-nano") req.reply(createChatTitleResponse("openai", "Test Chat")) return } + expect(req.body.model).to.equal("gpt-5.4") req.alias = "reasoningChat" req.reply( createResponse( @@ -340,6 +344,51 @@ describe("ai provider setup flows", () => { }) }) + it("keeps the provider identity of identical built-in model ids", () => { + cy.loadConsoleWithAuth(false, { + "ai.assistant.settings": JSON.stringify({ + modelValueFormat: 2, + selectedModel: "anthropic:shared-model", + providers: { + anthropic: { + apiKey: "test-anthropic-key", + enabledModels: ["anthropic:shared-model"], + grantSchemaAccess: false, + }, + openai: { + apiKey: "test-openai-key", + enabledModels: ["openai:shared-model"], + grantSchemaAccess: false, + }, + }, + }), + }) + + // Both rows keep the raw model name as their visible label. + cy.getByDataHook("ai-settings-model-dropdown").click() + cy.getByDataHook("ai-settings-model-item") + .should("have.length", 2) + .find("[data-hook='ai-settings-model-item-label']") + .each(($label) => { + expect($label.text()).to.equal("Shared Model") + }) + + // Selecting the OpenAI row persists its provider-qualified identity. + cy.getByDataHook("ai-settings-model-item").eq(1).click() + cy.window().then((win) => { + expect(readAiSettings(win).selectedModel).to.equal("openai:shared-model") + }) + + // Execution uses that identity to select the OpenAI provider. + interceptAIChatRequest("openai", "overlappingOpenAIModel") + cy.getByDataHook("ai-chat-button").click() + cy.getByDataHook("chat-input-textarea").type("hello") + cy.getByDataHook("chat-send-button").click() + cy.wait("@overlappingOpenAIModel") + .its("request.body.model") + .should("equal", "shared-model") + }) + it("manages the OpenAI provider lifecycle from the settings modal", () => { // Given a console already configured with Anthropic cy.loadConsoleWithAuth(false, getAnthropicConfiguredSettings()) @@ -375,11 +424,11 @@ describe("ai provider setup flows", () => { cy.window().then((win) => { const settings = readAiSettings(win) expect(settings.providers.openai.enabledModels).to.deep.equal([ - "gpt-5.4", - "gpt-5-mini", - "gpt-5", - "gpt-5-2025-08-06", - "gpt-5-nano", + "openai:gpt-5.4", + "openai:gpt-5-mini", + "openai:gpt-5", + "openai:gpt-5-2025-08-06", + "openai:gpt-5-nano", ]) }) @@ -414,7 +463,7 @@ describe("ai provider setup flows", () => { cy.getByDataHook("manage-models-save").should("not.exist") cy.window().then((win) => { const settings = readAiSettings(win) - expect(settings.selectedModel).to.equal("gpt-5-mini") + expect(settings.selectedModel).to.equal("openai:gpt-5-mini") expect(settings.providers.openai.grantSchemaAccess).to.equal(true) expect(settings.providers.openai.read).to.equal(false) expect(settings.providers.openai.write).to.equal(false) @@ -455,10 +504,10 @@ describe("ai provider setup flows", () => { cy.window().then((win) => { const settings = readAiSettings(win) expect(settings.providers.openai.enabledModels).to.deep.equal([ - "gpt-5-mini", - "gpt-5", - "gpt-5-2025-08-06", - "gpt-5-nano", + "openai:gpt-5-mini", + "openai:gpt-5", + "openai:gpt-5-2025-08-06", + "openai:gpt-5-nano", ]) }) }) @@ -520,9 +569,9 @@ describe("ai provider setup flows", () => { cy.window().then((win) => { const settings = readAiSettings(win) expect(settings.providers.openai.enabledModels).to.deep.equal([ - "gpt-5.4", - "my-proxy-model", - "gpt-5", + "openai:gpt-5.4", + "openai:my-proxy-model", + "openai:gpt-5", ]) }) @@ -581,7 +630,7 @@ describe("ai provider setup flows", () => { // And the saved configuration is untouched cy.window().then((win) => { expect(readAiSettings(win).providers.openai.enabledModels).to.deep.equal([ - "gpt-5.4", + "openai:gpt-5.4", ]) }) }) @@ -646,9 +695,11 @@ describe("ai provider setup flows", () => { cy.window().then((win) => { const settings = readAiSettings(win) expect(settings.providers.anthropic.enabledModels).to.deep.equal([ - "claude-haiku-4-5", + "anthropic:claude-haiku-4-5", + ]) + expect(settings.providers.openai.enabledModels).to.deep.equal([ + "openai:gpt-5.4", ]) - expect(settings.providers.openai.enabledModels).to.deep.equal(["gpt-5.4"]) }) // When a validation response arrives for a key that was already edited diff --git a/e2e/utils/aiAssistant.js b/e2e/utils/aiAssistant.js index 430c32e76..d9c89be96 100644 --- a/e2e/utils/aiAssistant.js +++ b/e2e/utils/aiAssistant.js @@ -22,11 +22,12 @@ const CUSTOM_PROVIDER_DEFAULTS = { function getOpenAIConfiguredSettings(schemaAccess = true) { return { "ai.assistant.settings": JSON.stringify({ - selectedModel: "gpt-5-mini", + modelValueFormat: 2, + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "test-openai-key", - enabledModels: ["gpt-5-mini", "gpt-5"], + enabledModels: ["openai:gpt-5-mini", "openai:gpt-5"], grantSchemaAccess: schemaAccess, }, }, @@ -48,11 +49,12 @@ function getOpenAIPermissionedSettings({ }) { return { "ai.assistant.settings": JSON.stringify({ - selectedModel: "gpt-5-mini", + modelValueFormat: 2, + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "test-openai-key", - enabledModels: ["gpt-5-mini", "gpt-5"], + enabledModels: ["openai:gpt-5-mini", "openai:gpt-5"], grantSchemaAccess, read, write, @@ -65,11 +67,15 @@ function getOpenAIPermissionedSettings({ function getAnthropicConfiguredSettings(schemaAccess = true) { return { "ai.assistant.settings": JSON.stringify({ - selectedModel: "claude-sonnet-4-5", + modelValueFormat: 2, + selectedModel: "anthropic:claude-sonnet-4-5", providers: { anthropic: { apiKey: "test-anthropic-key", - enabledModels: ["claude-sonnet-4-5", "claude-opus-4-5"], + enabledModels: [ + "anthropic:claude-sonnet-4-5", + "anthropic:claude-opus-4-5", + ], grantSchemaAccess: schemaAccess, }, }, @@ -101,6 +107,7 @@ function getCustomProviderConfiguredSettings(config = {}, mergeWith = null) { const settings = { ...baseSettings, + modelValueFormat: 2, selectedModel: enabledModels[0], customProviders: { ...(baseSettings.customProviders || {}), diff --git a/src/components/SetupAIAssistant/ConfigurationModal.tsx b/src/components/SetupAIAssistant/ConfigurationModal.tsx index a13edb8a0..95c3dabdc 100644 --- a/src/components/SetupAIAssistant/ConfigurationModal.tsx +++ b/src/components/SetupAIAssistant/ConfigurationModal.tsx @@ -17,7 +17,7 @@ import { filterOpenAiChatModels, formatModelLabel, getAllProviders, - makeCustomModelValue, + makeModelValue, sortModelsNewestFirst, type ProviderId, type ProviderModel, @@ -560,6 +560,9 @@ export const ConfigurationModal = ({ const handleComplete = () => { const models = effectiveEnabledModels() if (!selectedProvider || models.length === 0) return + const modelValues = models.map((model) => + makeModelValue(selectedProvider, model), + ) void trackEvent(ConsoleEvent.AI_PROVIDER_CONFIGURE, { name: selectedProvider, @@ -574,12 +577,12 @@ export const ConfigurationModal = ({ const newSettings = { ...aiAssistantSettings, - selectedModel: models[0], + selectedModel: modelValues[0], providers: { ...aiAssistantSettings.providers, [selectedProvider]: buildProviderSettings({ apiKey, - enabledModels: models, + enabledModels: modelValues, permissions, modelLabels: metadata?.modelLabels, utilityModel: metadata?.utilityModel, @@ -671,7 +674,7 @@ export const ConfigurationModal = ({ const handleCustomProviderSave = useCallback( (providerId: string, definition: CustomProviderDefinition) => { const newEnabledModels = definition.models.map((m) => - makeCustomModelValue(providerId, m), + makeModelValue(providerId, m), ) const newSettings = { diff --git a/src/components/SetupAIAssistant/SettingsModal.tsx b/src/components/SetupAIAssistant/SettingsModal.tsx index 9950c331e..3e54075b5 100644 --- a/src/components/SetupAIAssistant/SettingsModal.tsx +++ b/src/components/SetupAIAssistant/SettingsModal.tsx @@ -22,7 +22,7 @@ import { getAllProviders, getAllModelOptions, getApiKey, - makeCustomModelValue, + makeModelValue, stripModelNamespace, formatModelLabel, buildProviderSettings, @@ -32,6 +32,7 @@ import { type ProviderModel, getNextModel, getProviderName, + getModelListingErrorMessage, } from "../../utils/ai" import { createProvider } from "../../utils/ai/registry" import type { @@ -646,19 +647,11 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { if (isStale()) return const aiProvider = createProvider(provider, apiKey, localSettings) const classified = aiProvider.classifyError(err, () => {}) - if (!isBuiltin && classified.type !== "invalid_key") { - // Custom endpoints often lack a model listing — the key may still work. - setValidationState((prev) => ({ ...prev, [provider]: "validated" })) - setValidatedApiKeys((prev) => ({ ...prev, [provider]: true })) - return - } setValidationState((prev) => ({ ...prev, [provider]: "error" })) + setValidatedApiKeys((prev) => ({ ...prev, [provider]: false })) setValidationErrors((prev) => ({ ...prev, - [provider]: - classified.type === "invalid_key" - ? "Invalid API key" - : classified.message, + [provider]: getModelListingErrorMessage(err, classified), })) } }, @@ -736,7 +729,6 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { updatedSettings.selectedModel, enabledModels, updatedSettings, - aiAssistantSettings, ) updatedSettings.selectedModel = nextModel || undefined @@ -810,7 +802,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { const handleCustomProviderSave = useCallback( (providerId: string, definition: CustomProviderDefinition) => { const newEnabledModels = definition.models.map((m) => - makeCustomModelValue(providerId, m), + makeModelValue(providerId, m), ) setLocalCustomProviders((prev) => ({ @@ -867,7 +859,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { const handleManageModelsSave = useCallback( (providerId: string, definition: CustomProviderDefinition) => { const newModelValues = definition.models.map((m) => - makeCustomModelValue(providerId, m), + makeModelValue(providerId, m), ) // Update local custom providers — only override models and contextWindow, @@ -884,7 +876,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { // Determine which models are truly new (not in the previous model list) const oldModelValues = ( localCustomProviders[providerId]?.models || [] - ).map((m) => makeCustomModelValue(providerId, m)) + ).map((m) => makeModelValue(providerId, m)) const trulyNew = newModelValues.filter((m) => !oldModelValues.includes(m)) // Local state: respect unsaved checkbox toggles, add truly new as enabled @@ -974,9 +966,12 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { const handleBuiltinModelsSave = useCallback( (providerId: string, result: BuiltinModelsResult) => { builtinModelsSavedRef.current = true + const modelValues = result.enabledModels.map((model) => + makeModelValue(providerId, model), + ) setEnabledModels((prev) => ({ ...prev, - [providerId]: result.enabledModels, + [providerId]: modelValues, })) setModelLabels((prev) => ({ ...prev, [providerId]: result.modelLabels })) setUtilityModels((prev) => ({ @@ -1000,7 +995,7 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { ...aiAssistantSettings.providers, [providerId]: buildProviderSettings({ apiKey: apiKeys[providerId] ?? "", - enabledModels: result.enabledModels, + enabledModels: modelValues, permissions: persistedPermissions, modelLabels: result.modelLabels, utilityModel: result.utilityModel, @@ -1021,7 +1016,6 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { updatedSettings.selectedModel, persistedEnabledModels, updatedSettings, - aiAssistantSettings, ) || undefined updateSettings(StoreKey.AI_ASSISTANT_SETTINGS, updatedSettings) @@ -1051,9 +1045,9 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { ) const labelForModel = (provider: ProviderId, value: string) => { - if (!BUILTIN_PROVIDERS[provider]) - return stripModelNamespace(value, provider) - return modelLabels[provider]?.[value] ?? formatModelLabel(value) + const modelId = stripModelNamespace(value, provider) + if (!BUILTIN_PROVIDERS[provider]) return modelId + return modelLabels[provider]?.[modelId] ?? formatModelLabel(modelId) } const allProviders = useMemo( @@ -1320,13 +1314,17 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { {enabledModelsForProvider.map((value) => { const label = labelForModel(selectedProvider, value) + const modelId = stripModelNamespace( + value, + selectedProvider, + ) return ( {label} - {!isCustomProvider && label !== value && ( + {!isCustomProvider && label !== modelId && ( - {value} + {modelId} )} @@ -1447,7 +1445,9 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { onOpenChange={handleBuiltinModelsOpenChange} providerId={manageModelsProvider} apiKey={apiKeys[manageModelsProvider] ?? ""} - enabledModels={enabledModels[manageModelsProvider] ?? []} + enabledModels={(enabledModels[manageModelsProvider] ?? []).map( + (model) => stripModelNamespace(model, manageModelsProvider), + )} initialListing={validationListings[manageModelsProvider]} onSave={handleBuiltinModelsSave} /> diff --git a/src/providers/LocalStorageProvider/index.tsx b/src/providers/LocalStorageProvider/index.tsx index d2c73875a..8c66a5be2 100644 --- a/src/providers/LocalStorageProvider/index.tsx +++ b/src/providers/LocalStorageProvider/index.tsx @@ -32,6 +32,7 @@ import React, { useRef, } from "react" import { getValue, setValue } from "../../utils/localStorage" +import { migrateLocalStorage } from "../../utils/localStorage/migrate" import { StoreKey } from "../../utils/localStorage/types" import { parseInteger, @@ -41,6 +42,7 @@ import { } from "./utils" import type { MaxColumnWidth } from "../../components/ResultGrid/types" import { + AI_MODEL_VALUE_FORMAT, AiAssistantSettings, LocalConfig, SettingsType, @@ -49,10 +51,10 @@ import { NotebookOnboarding, RunWithSelectionMode, } from "./types" -import { reconcileSettings } from "../../utils/ai/settings" import { onReasoningUnsupported } from "../../utils/ai/reasoningFallback" export const DEFAULT_AI_ASSISTANT_SETTINGS: AiAssistantSettings = { + modelValueFormat: AI_MODEL_VALUE_FORMAT, providers: {}, } @@ -158,24 +160,21 @@ type ContextProps = { } const getAiAssistantSettings = (): AiAssistantSettings => { - const stored = getValue(StoreKey.AI_ASSISTANT_SETTINGS) - if (stored) { - try { + try { + const stored = getValue(StoreKey.AI_ASSISTANT_SETTINGS) + if (stored) { const parsed = JSON.parse(stored) as AiAssistantSettings - const reconciled = reconcileSettings({ + return { + modelValueFormat: parsed.modelValueFormat, selectedModel: parsed.selectedModel, providers: parsed.providers || {}, ...(parsed.customProviders && { customProviders: parsed.customProviders, }), - }) - if (JSON.stringify(reconciled) !== stored) { - setValue(StoreKey.AI_ASSISTANT_SETTINGS, JSON.stringify(reconciled)) } - return reconciled - } catch (e) { - return defaultConfig.aiAssistantSettings } + } catch { + return defaultConfig.aiAssistantSettings } return defaultConfig.aiAssistantSettings } @@ -278,7 +277,10 @@ export const LocalStorageProvider = ({ useState(getLeftPanelState()) const [aiAssistantSettings, setAiAssistantSettings] = - useState(getAiAssistantSettings()) + useState(() => { + if (!migrateLocalStorage()) return defaultConfig.aiAssistantSettings + return getAiAssistantSettings() + }) const [aiChatPanelWidth, setAiChatPanelWidth] = useState( parseInteger( diff --git a/src/providers/LocalStorageProvider/types.ts b/src/providers/LocalStorageProvider/types.ts index 55a000006..cfe0e39d5 100644 --- a/src/providers/LocalStorageProvider/types.ts +++ b/src/providers/LocalStorageProvider/types.ts @@ -24,7 +24,11 @@ export type CustomProviderDefinition = { write?: boolean } +export const AI_MODEL_VALUE_FORMAT = 2 as const + export type AiAssistantSettings = { + /** Version 2 stores globally referenced models as `providerId:modelId`. */ + modelValueFormat?: typeof AI_MODEL_VALUE_FORMAT selectedModel?: string providers: Partial> customProviders?: Record diff --git a/src/utils/ai/anthropicProvider.namespace.test.ts b/src/utils/ai/anthropicProvider.namespace.test.ts new file mode 100644 index 000000000..aec776900 --- /dev/null +++ b/src/utils/ai/anthropicProvider.namespace.test.ts @@ -0,0 +1,85 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" +import { createAnthropicProvider } from "./anthropicProvider" + +const { createMock, streamMock, countTokensMock } = vi.hoisted(() => ({ + createMock: vi.fn(), + streamMock: vi.fn(), + countTokensMock: vi.fn(), +})) + +vi.mock("@anthropic-ai/sdk", () => ({ + default: class MockAnthropic { + messages = { + create: createMock, + stream: streamMock, + countTokens: countTokensMock, + } + models = { list: vi.fn() } + }, +})) + +const message = { + id: "msg_1", + type: "message", + role: "assistant", + model: "claude-test", + content: [{ type: "text", text: "done" }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, +} + +const summaryStream = () => + (async function* () { + await Promise.resolve() + yield { + type: "content_block_delta", + delta: { type: "text_delta", text: "summary" }, + } + })() + +const capturedModels = (mock: { + mock: { calls: unknown[][] } +}): Array => + mock.mock.calls.map( + (call) => (call[0] as { model?: string } | undefined)?.model, + ) + +describe("anthropic provider model namespaces", () => { + beforeEach(() => { + createMock.mockReset().mockResolvedValue(message) + streamMock.mockReset().mockImplementation(summaryStream) + countTokensMock.mockReset().mockResolvedValue({ input_tokens: 7 }) + }) + + it("sends raw model ids at every Anthropic request boundary", async () => { + const provider = createAnthropicProvider("sk-test") + const model = "anthropic:claude-test" + + await provider.executeFlow({ + model, + config: { + systemInstructions: "system", + initialUserContent: "hello", + }, + modelToolsClient: {} as never, + tools: [], + setStatus: () => {}, + }) + await provider.generateTitle({ model, prompt: "title" }) + await provider.generateSummary({ + model, + systemPrompt: "system", + userMessage: "summarize", + }) + await provider.countTokens({ + model, + systemPrompt: "system", + messages: [{ role: "user", content: "hello" }], + }) + + expect(capturedModels(createMock)).toEqual(["claude-test", "claude-test"]) + expect(capturedModels(streamMock)).toEqual(["claude-test"]) + expect(capturedModels(countTokensMock)).toEqual(["claude-test"]) + }) +}) diff --git a/src/utils/ai/index.ts b/src/utils/ai/index.ts index 21770b947..8af428fc2 100644 --- a/src/utils/ai/index.ts +++ b/src/utils/ai/index.ts @@ -38,7 +38,7 @@ export { getProviderContextWindow, getApiKey, buildProviderSettings, - makeCustomModelValue, + makeModelValue, stripModelNamespace, isAiAssistantConfigured, canUseAiAssistant, @@ -60,3 +60,4 @@ export { UTILITY_MODEL_TIERS, } from "./modelCatalog" export type { ProviderModel } from "./modelCatalog" +export { getModelListingErrorMessage } from "./modelListingError" diff --git a/src/utils/ai/modelListingError.test.ts b/src/utils/ai/modelListingError.test.ts new file mode 100644 index 000000000..587e973ec --- /dev/null +++ b/src/utils/ai/modelListingError.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest" +import { getModelListingErrorMessage } from "./modelListingError" +import type { AiAssistantAPIError } from "./aiAssistant" + +const classified = ( + type: AiAssistantAPIError["type"] = "unknown", +): AiAssistantAPIError => ({ type, message: "Provider error" }) + +describe("getModelListingErrorMessage", () => { + it.each([ + [401, "Invalid API key"], + [403, "This API key does not have permission to list models"], + [404, "This provider does not support model listing"], + [405, "This provider does not support model listing"], + [429, "The provider rate limit was reached"], + [500, "The provider is temporarily unavailable"], + [503, "The provider is temporarily unavailable"], + ])("maps HTTP %s to a specific message", (status, expected) => { + expect(getModelListingErrorMessage({ status }, classified())).toContain( + expected, + ) + }) + + it("describes connection failures without claiming validation succeeded", () => { + expect( + getModelListingErrorMessage( + new Error("fetch failed"), + classified("network"), + ), + ).toBe( + "Could not reach the provider. Check its URL and your network connection.", + ) + }) + + it("preserves the provider message for other failures", () => { + expect( + getModelListingErrorMessage( + { status: 422 }, + { type: "unknown", message: "Unsupported request" }, + ), + ).toBe("Unsupported request") + }) +}) diff --git a/src/utils/ai/modelListingError.ts b/src/utils/ai/modelListingError.ts new file mode 100644 index 000000000..4fe53d750 --- /dev/null +++ b/src/utils/ai/modelListingError.ts @@ -0,0 +1,36 @@ +import type { AiAssistantAPIError } from "./aiAssistant" + +const getHttpStatus = (error: unknown): number | null => { + if (typeof error !== "object" || error === null || !("status" in error)) { + return null + } + const status = (error as { status?: unknown }).status + return typeof status === "number" ? status : null +} + +export const getModelListingErrorMessage = ( + error: unknown, + classified: AiAssistantAPIError, +): string => { + const status = getHttpStatus(error) + + if (status === 401 || classified.type === "invalid_key") { + return "Invalid API key" + } + if (status === 403) { + return "This API key does not have permission to list models" + } + if (status === 404 || status === 405) { + return "This provider does not support model listing. Configure its models manually." + } + if (status === 429 || classified.type === "rate_limit") { + return "The provider rate limit was reached. Please try again later." + } + if (status !== null && status >= 500) { + return "The provider is temporarily unavailable. Please try again later." + } + if (classified.type === "network") { + return "Could not reach the provider. Check its URL and your network connection." + } + return classified.message +} diff --git a/src/utils/ai/openaiProvider.fallback.test.ts b/src/utils/ai/openaiProvider.fallback.test.ts index 66bf451e4..cc6db3f5b 100644 --- a/src/utils/ai/openaiProvider.fallback.test.ts +++ b/src/utils/ai/openaiProvider.fallback.test.ts @@ -72,7 +72,7 @@ describe("openai reasoning fallback", () => { // When generating a summary const text = await provider.generateSummary({ - model: "gpt-4o", + model: "openai:gpt-4o", systemPrompt: "sys", userMessage: "user", }) @@ -84,6 +84,8 @@ describe("openai reasoning fallback", () => { effort: "high", summary: "auto", }) + expect(requestBodies()[0].model).toBe("gpt-4o") + expect(requestBodies()[1].model).toBe("gpt-4o") expect("reasoning" in requestBodies()[1]).toBe(false) // And the downgrade was surfaced and reported exactly once diff --git a/src/utils/ai/settings.test.ts b/src/utils/ai/settings.test.ts index ffe97fbd5..ab0eb77ba 100644 --- a/src/utils/ai/settings.test.ts +++ b/src/utils/ai/settings.test.ts @@ -1,6 +1,5 @@ import { describe, it, expect } from "vitest" import { - reconcileSettings, getSelectedModel, getAiPermissions, getAllModelOptions, @@ -8,9 +7,16 @@ import { getUtilityModel, providerForModel, buildListingMetadata, + makeModelValue, + parseModelValue, + stripModelNamespace, } from "./settings" +import { migrateLocalStorage } from "../localStorage/migrate" -import type { AiAssistantSettings } from "../../providers/LocalStorageProvider/types" +import { + AI_MODEL_VALUE_FORMAT, + type AiAssistantSettings, +} from "../../providers/LocalStorageProvider/types" const makeSettings = ( overrides: Partial = {}, @@ -19,7 +25,122 @@ const makeSettings = ( ...overrides, }) -describe("reconcileSettings", () => { +const migrateSettings = ( + settings: AiAssistantSettings, +): AiAssistantSettings => { + let stored = JSON.stringify(settings) + migrateLocalStorage({ + getItem: () => stored, + setItem: (_key, value) => { + stored = value + }, + }) + return JSON.parse(stored) as AiAssistantSettings +} + +describe("migrateLocalStorage", () => { + it("isolates local-storage read and write failures", () => { + expect( + migrateLocalStorage({ + getItem: () => { + throw new Error("storage unavailable") + }, + setItem: () => undefined, + }), + ).toBe(false) + + expect( + migrateLocalStorage({ + getItem: () => + JSON.stringify({ + providers: {}, + }), + setItem: () => { + throw new Error("storage quota exceeded") + }, + }), + ).toBe(false) + }) + + it.each([AI_MODEL_VALUE_FORMAT, 3])( + "does not touch settings carrying format version %s", + (modelValueFormat) => { + const stored = JSON.stringify({ + modelValueFormat, + selectedModel: "openai:openai:foo", + providers: { + openai: { + apiKey: "sk-test", + enabledModels: ["openai:openai:foo"], + grantSchemaAccess: false, + }, + }, + futureField: "preserved byte-for-byte", + }) + let writes = 0 + + migrateLocalStorage({ + getItem: () => stored, + setItem: () => { + writes += 1 + }, + }) + + expect(writes).toBe(0) + }, + ) + + it("migrates built-in model values and utility models to provider-qualified storage", () => { + const settings = makeSettings({ + selectedModel: "gpt-5.4", + providers: { + openai: { + apiKey: "sk-test", + enabledModels: ["gpt-5.4", "gpt-5-mini"], + utilityModel: "gpt-5-mini", + modelLabels: { "gpt-5.4": "GPT-5.4" }, + grantSchemaAccess: false, + }, + }, + }) + + const result = migrateSettings(settings) + + expect(result.modelValueFormat).toBe(AI_MODEL_VALUE_FORMAT) + expect(result.selectedModel).toBe("openai:gpt-5.4") + expect(result.providers.openai!.enabledModels).toEqual([ + "openai:gpt-5.4", + "openai:gpt-5-mini", + ]) + expect(result.providers.openai!.utilityModel).toBe("openai:gpt-5-mini") + expect(result.providers.openai!.modelLabels).toEqual({ + "gpt-5.4": "GPT-5.4", + }) + }) + + it("preserves a literal provider prefix inside a legacy model id", () => { + const result = migrateSettings( + makeSettings({ + selectedModel: "openai:foo", + providers: { + openai: { + apiKey: "sk-test", + enabledModels: ["openai:foo"], + grantSchemaAccess: false, + }, + }, + }), + ) + + expect(result.selectedModel).toBe("openai:openai:foo") + expect(result.providers.openai!.enabledModels).toEqual([ + "openai:openai:foo", + ]) + expect(stripModelNamespace(result.selectedModel!, "openai")).toBe( + "openai:foo", + ) + }) + it("keeps built-in model ids it does not recognize", () => { // Given enabled models that no fixed list knows about const settings = makeSettings({ @@ -31,12 +152,12 @@ describe("reconcileSettings", () => { }, }, }) - // When settings reconcile - const result = reconcileSettings(settings) - // Then availability is the picker's job, not reconcile's + // When storage migrates + const result = migrateSettings(settings) + // Then availability is the picker's job, not the migration's expect(result.providers.openai!.enabledModels).toEqual([ - "gpt-7", - "gpt-5-mini", + "openai:gpt-7", + "openai:gpt-5-mini", ]) }) @@ -54,10 +175,10 @@ describe("reconcileSettings", () => { }, }, }) - const result = reconcileSettings(settings) + const result = migrateSettings(settings) expect(result.providers.openai!.enabledModels).toEqual([ - "gpt-5.4", - "gpt-5-mini", + "openai:gpt-5.4", + "openai:gpt-5-mini", ]) }) @@ -72,9 +193,11 @@ describe("reconcileSettings", () => { }, }, }) - const result = reconcileSettings(settings) - expect(result.providers.anthropic!.enabledModels).toEqual(["claude-sonnet"]) - expect(result.selectedModel).toBe("claude-sonnet") + const result = migrateSettings(settings) + expect(result.providers.anthropic!.enabledModels).toEqual([ + "anthropic:claude-sonnet", + ]) + expect(result.selectedModel).toBe("anthropic:claude-sonnet") }) it("folds a selected high variant into reasoningEffort", () => { @@ -89,11 +212,11 @@ describe("reconcileSettings", () => { }, }, }) - // When settings reconcile - const result = reconcileSettings(settings) + // When storage migrates + const result = migrateSettings(settings) // Then the provider runs on High and the selection is the plain id expect(result.providers.openai!.reasoningEffort).toBe("high") - expect(result.selectedModel).toBe("gpt-5.4") + expect(result.selectedModel).toBe("openai:gpt-5.4") }) it("migrates medium and low variant users to the provider default", () => { @@ -107,9 +230,9 @@ describe("reconcileSettings", () => { }, }, }) - const result = reconcileSettings(settings) + const result = migrateSettings(settings) expect(result.providers.openai!.reasoningEffort).toBeUndefined() - expect(result.selectedModel).toBe("gpt-5.4") + expect(result.selectedModel).toBe("openai:gpt-5.4") }) it("removes custom models missing from their provider definition", () => { @@ -131,7 +254,7 @@ describe("reconcileSettings", () => { }, }, }) - const result = reconcileSettings(settings) + const result = migrateSettings(settings) expect(result.providers["custom-1"]!.enabledModels).toEqual([ "custom-1:llm-a", ]) @@ -163,7 +286,7 @@ describe("reconcileSettings", () => { }, }, }) - const result = reconcileSettings(settings) + const result = migrateSettings(settings) expect(result.providers["custom-1"]!.enabledModels).toEqual(enabledModels) expect(result.selectedModel).toBe(enabledModels[0]) }) @@ -184,8 +307,8 @@ describe("reconcileSettings", () => { }, }, }) - const once = reconcileSettings(settings) - const twice = reconcileSettings(once) + const once = migrateSettings(settings) + const twice = migrateSettings(once) expect(twice).toEqual(once) }) @@ -204,7 +327,7 @@ describe("reconcileSettings", () => { string > settingsWithFutureField.futureField = "preserved" - const result = reconcileSettings(settings) + const result = migrateSettings(settings) expect((result as unknown as Record).futureField).toBe( "preserved", ) @@ -221,13 +344,13 @@ describe("reconcileSettings", () => { }, }, }) - const result = reconcileSettings(settings) - expect(result.selectedModel).toEqual("gpt-5-mini") + const result = migrateSettings(settings) + expect(result.selectedModel).toEqual("openai:gpt-5-mini") }) it("preserves selectedModel if it is in enabledModels", () => { const settings = makeSettings({ - selectedModel: "gpt-5-mini", + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "sk-test", @@ -236,13 +359,13 @@ describe("reconcileSettings", () => { }, }, }) - const result = reconcileSettings(settings) - expect(result.selectedModel).toBe("gpt-5-mini") + const result = migrateSettings(settings) + expect(result.selectedModel).toBe("openai:gpt-5-mini") }) it("handles empty providers gracefully", () => { const settings = makeSettings({ providers: {} }) - const result = reconcileSettings(settings) + const result = migrateSettings(settings) expect(result.providers).toEqual({}) }) @@ -257,7 +380,7 @@ describe("reconcileSettings", () => { }, }) const originalModels = [...settings.providers.openai!.enabledModels] - reconcileSettings(settings) + migrateSettings(settings) expect(settings.providers.openai!.enabledModels).toEqual(originalModels) }) }) @@ -265,30 +388,30 @@ describe("reconcileSettings", () => { describe("getSelectedModel", () => { it("returns selectedModel when it is in enabledModels", () => { const settings = makeSettings({ - selectedModel: "gpt-5-mini", + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini", "gpt-5"], + enabledModels: ["openai:gpt-5-mini", "openai:gpt-5"], grantSchemaAccess: false, }, }, }) - expect(getSelectedModel(settings)).toBe("gpt-5-mini") + expect(getSelectedModel(settings)).toBe("openai:gpt-5-mini") }) it("falls back to the first enabled model", () => { const settings = makeSettings({ - selectedModel: "claude-sonnet-4-5", + selectedModel: "anthropic:claude-sonnet-4-5", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini"], + enabledModels: ["openai:gpt-5-mini"], grantSchemaAccess: false, }, }, }) - expect(getSelectedModel(settings)).toBe("gpt-5-mini") + expect(getSelectedModel(settings)).toBe("openai:gpt-5-mini") }) it("returns null when no models are enabled", () => { @@ -298,13 +421,49 @@ describe("getSelectedModel", () => { }) describe("getAllModelOptions", () => { + it("keeps identical built-in model ids distinct while showing raw labels", () => { + const settings = makeSettings({ + modelValueFormat: AI_MODEL_VALUE_FORMAT, + selectedModel: "openai:shared-model", + providers: { + anthropic: { + apiKey: "sk-ant", + enabledModels: ["anthropic:shared-model"], + grantSchemaAccess: false, + }, + openai: { + apiKey: "sk-openai", + enabledModels: ["openai:shared-model"], + grantSchemaAccess: false, + }, + }, + }) + + const options = getAllModelOptions(settings) + + expect(options).toEqual([ + { + label: "Shared Model", + value: "anthropic:shared-model", + provider: "anthropic", + }, + { + label: "Shared Model", + value: "openai:shared-model", + provider: "openai", + }, + ]) + expect(new Set(options.map((option) => option.value)).size).toBe(2) + expect(providerForModel(options[1].value, settings)).toBe("openai") + }) + it("builds options from enabled models with stored labels", () => { // Given a provider with one stored label and one without const settings = makeSettings({ providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5.4", "gpt-5-mini"], + enabledModels: ["openai:gpt-5.4", "openai:gpt-5-mini"], grantSchemaAccess: false, modelLabels: { "gpt-5.4": "GPT-5.4 (Custom)" }, }, @@ -314,8 +473,16 @@ describe("getAllModelOptions", () => { const options = getAllModelOptions(settings) // Then the stored label wins and the formatter fills the gap expect(options).toEqual([ - { label: "GPT-5.4 (Custom)", value: "gpt-5.4", provider: "openai" }, - { label: "GPT 5 Mini", value: "gpt-5-mini", provider: "openai" }, + { + label: "GPT-5.4 (Custom)", + value: "openai:gpt-5.4", + provider: "openai", + }, + { + label: "GPT 5 Mini", + value: "openai:gpt-5-mini", + provider: "openai", + }, ]) }) @@ -338,17 +505,47 @@ describe("getAllModelOptions", () => { }) describe("providerForModel", () => { + it("parses built-in and custom provider-qualified values", () => { + const customProviders = { + "custom-1": { + type: "openai-chat-completions" as const, + name: "Test", + baseURL: "http://localhost:11434/v1", + contextWindow: 100_000, + models: ["llm-a"], + }, + } + + expect(parseModelValue("openai:gpt-5.4", customProviders)).toEqual({ + providerId: "openai", + rawModel: "gpt-5.4", + }) + expect(parseModelValue("custom-1:llm-a", customProviders)).toEqual({ + providerId: "custom-1", + rawModel: "llm-a", + }) + expect(makeModelValue("openai", "openai:foo")).toBe("openai:openai:foo") + expect(stripModelNamespace("openai:openai:foo", "openai")).toBe( + "openai:foo", + ) + expect(stripModelNamespace("anthropic:shared-model", "openai")).toBe( + "anthropic:shared-model", + ) + }) + it("finds the built-in provider that enabled the model", () => { const settings = makeSettings({ providers: { anthropic: { apiKey: "sk-test", - enabledModels: ["claude-sonnet-5"], + enabledModels: ["anthropic:claude-sonnet-5"], grantSchemaAccess: false, }, }, }) - expect(providerForModel("claude-sonnet-5", settings)).toBe("anthropic") + expect(providerForModel("anthropic:claude-sonnet-5", settings)).toBe( + "anthropic", + ) expect(providerForModel("gpt-5-mini", settings)).toBeNull() }) @@ -372,7 +569,7 @@ describe("providerForModel", () => { it("routes colon-containing listed ids to the built-in provider that enabled them", () => { // Given an OpenAI fine-tune id enabled under the built-in provider - const fineTune = "ft:gpt-4o-mini-2024-07-18:acme::BxK9pQ2r" + const fineTune = "openai:ft:gpt-4o-mini-2024-07-18:acme::BxK9pQ2r" const settings = makeSettings({ providers: { openai: { @@ -390,8 +587,10 @@ describe("providerForModel", () => { describe("getNextModel", () => { it("keeps the current model while it stays enabled", () => { expect( - getNextModel("gpt-5-mini", { openai: ["gpt-5.4", "gpt-5-mini"] }), - ).toBe("gpt-5-mini") + getNextModel("openai:gpt-5-mini", { + openai: ["openai:gpt-5.4", "openai:gpt-5-mini"], + }), + ).toBe("openai:gpt-5-mini") }) it("takes the first enabled model of any provider when the current one is gone", () => { @@ -399,48 +598,37 @@ describe("getNextModel", () => { providers: { anthropic: { apiKey: "sk-test", - enabledModels: ["claude-sonnet-5"], + enabledModels: ["anthropic:claude-sonnet-5"], grantSchemaAccess: false, }, }, }) expect( - getNextModel("gpt-5-mini", { anthropic: ["claude-sonnet-5"] }, settings), - ).toBe("claude-sonnet-5") + getNextModel( + "openai:gpt-5-mini", + { anthropic: ["anthropic:claude-sonnet-5"] }, + settings, + ), + ).toBe("anthropic:claude-sonnet-5") }) it("returns null when nothing is enabled", () => { - expect(getNextModel("gpt-5-mini", {})).toBeNull() + expect(getNextModel("openai:gpt-5-mini", {})).toBeNull() }) it("stays on the outgoing model's provider when it still has models", () => { // Given gpt-5.4 was just disabled while OpenAI keeps gpt-5-mini enabled - const previousSettings = makeSettings({ - selectedModel: "gpt-5.4", - providers: { - openai: { - apiKey: "sk-test", - enabledModels: ["gpt-5.4", "gpt-5-mini"], - grantSchemaAccess: false, - }, - anthropic: { - apiKey: "sk-test", - enabledModels: ["claude-opus-5"], - grantSchemaAccess: false, - }, - }, - }) const updatedSettings = makeSettings({ - selectedModel: "gpt-5.4", + selectedModel: "openai:gpt-5.4", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini"], + enabledModels: ["openai:gpt-5-mini"], grantSchemaAccess: false, }, anthropic: { apiKey: "sk-test", - enabledModels: ["claude-opus-5"], + enabledModels: ["anthropic:claude-opus-5"], grantSchemaAccess: false, }, }, @@ -448,45 +636,47 @@ describe("getNextModel", () => { // When picking the next model const next = getNextModel( - "gpt-5.4", - { openai: ["gpt-5-mini"], anthropic: ["claude-opus-5"] }, + "openai:gpt-5.4", + { + openai: ["openai:gpt-5-mini"], + anthropic: ["anthropic:claude-opus-5"], + }, updatedSettings, - previousSettings, ) // Then it falls back within OpenAI instead of hopping to Anthropic - expect(next).toBe("gpt-5-mini") + expect(next).toBe("openai:gpt-5-mini") }) }) describe("getUtilityModel", () => { it("returns the persisted utility model for a built-in provider", () => { const settings = makeSettings({ - selectedModel: "gpt-5.4", + selectedModel: "openai:gpt-5.4", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5.4"], + enabledModels: ["openai:gpt-5.4"], grantSchemaAccess: false, - utilityModel: "gpt-5.6-luna", + utilityModel: "openai:gpt-5.6-luna", }, }, }) - expect(getUtilityModel("openai", settings)).toBe("gpt-5.6-luna") + expect(getUtilityModel("openai", settings)).toBe("openai:gpt-5.6-luna") }) it("falls back to the selected model when nothing is persisted", () => { const settings = makeSettings({ - selectedModel: "gpt-5.4", + selectedModel: "openai:gpt-5.4", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5.4"], + enabledModels: ["openai:gpt-5.4"], grantSchemaAccess: false, }, }, }) - expect(getUtilityModel("openai", settings)).toBe("gpt-5.4") + expect(getUtilityModel("openai", settings)).toBe("openai:gpt-5.4") }) it("uses the selected model for custom providers", () => { @@ -518,7 +708,7 @@ describe("getAiPermissions", () => { it("returns all-false when the selected model's provider has no settings", () => { const settings = makeSettings({ - selectedModel: "gpt-5-mini", + selectedModel: "openai:gpt-5-mini", providers: {}, }) expect(getAiPermissions(settings)).toEqual({ @@ -530,11 +720,11 @@ describe("getAiPermissions", () => { it("defaults read/write to false when only the legacy grantSchemaAccess is persisted", () => { const settings = makeSettings({ - selectedModel: "gpt-5-mini", + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini"], + enabledModels: ["openai:gpt-5-mini"], grantSchemaAccess: true, }, }, @@ -548,11 +738,11 @@ describe("getAiPermissions", () => { it("returns the three booleans verbatim when all are persisted on a built-in provider", () => { const settings = makeSettings({ - selectedModel: "gpt-5-mini", + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini"], + enabledModels: ["openai:gpt-5-mini"], grantSchemaAccess: true, read: true, write: true, @@ -600,11 +790,11 @@ describe("getAiPermissions", () => { it("returns false for read when grantSchemaAccess is true but read is explicitly false", () => { const settings = makeSettings({ - selectedModel: "gpt-5-mini", + selectedModel: "openai:gpt-5-mini", providers: { openai: { apiKey: "sk-test", - enabledModels: ["gpt-5-mini"], + enabledModels: ["openai:gpt-5-mini"], grantSchemaAccess: true, read: false, write: false, @@ -641,7 +831,7 @@ describe("buildListingMetadata", () => { "my-proxy-model": "my-proxy-model", }) // And the utility model comes from the cheap tier of the chat pool - expect(metadata.utilityModel).toBe("gpt-5.4-nano") + expect(metadata.utilityModel).toBe("openai:gpt-5.4-nano") }) it("prefers provider labels and haiku-tier utility for an Anthropic listing", () => { @@ -657,6 +847,6 @@ describe("buildListingMetadata", () => { // Then the stored label is the provider's and utility picks the haiku tier expect(metadata.modelLabels).toEqual({ "claude-opus-5": "Claude Opus 5" }) - expect(metadata.utilityModel).toBe("claude-haiku-4-5") + expect(metadata.utilityModel).toBe("anthropic:claude-haiku-4-5") }) }) diff --git a/src/utils/ai/settings.ts b/src/utils/ai/settings.ts index 7090ef4f9..d90697097 100644 --- a/src/utils/ai/settings.ts +++ b/src/utils/ai/settings.ts @@ -48,25 +48,28 @@ export type ModelOption = { provider: ProviderId } -const CUSTOM_MODEL_SEP = ":" +const MODEL_VALUE_SEP = ":" -export const makeCustomModelValue = ( +export const makeModelValue = ( providerId: ProviderId, modelId: string, -): string => `${providerId}${CUSTOM_MODEL_SEP}${modelId}` +): string => `${providerId}${MODEL_VALUE_SEP}${modelId}` export const parseModelValue = ( value: string, customProviders?: Record, -): { customProviderId: string; rawModel: string } | { rawModel: string } => { - const sepIndex = value.indexOf(CUSTOM_MODEL_SEP) +): { providerId: ProviderId; rawModel: string } | { rawModel: string } => { + const sepIndex = value.indexOf(MODEL_VALUE_SEP) if (sepIndex === -1) return { rawModel: value } const candidateProvider = value.slice(0, sepIndex) - if (!customProviders || !Object.hasOwn(customProviders, candidateProvider)) { + if ( + !Object.hasOwn(BUILTIN_PROVIDERS, candidateProvider) && + (!customProviders || !Object.hasOwn(customProviders, candidateProvider)) + ) { return { rawModel: value } } return { - customProviderId: candidateProvider, + providerId: candidateProvider, rawModel: value.slice(sepIndex + 1), } } @@ -75,17 +78,21 @@ export const stripModelNamespace = ( value: string, providerId: ProviderId, ): string => { - const prefix = `${providerId}${CUSTOM_MODEL_SEP}` + const prefix = `${providerId}${MODEL_VALUE_SEP}` return value.startsWith(prefix) ? value.slice(prefix.length) : value } export const getModelLabel = ( - modelId: string, + modelValue: string, providerId: ProviderId, settings?: AiAssistantSettings, -): string => - settings?.providers?.[providerId]?.modelLabels?.[modelId] ?? - formatModelLabel(modelId) +): string => { + const modelId = stripModelNamespace(modelValue, providerId) + return ( + settings?.providers?.[providerId]?.modelLabels?.[modelId] ?? + formatModelLabel(modelId) + ) +} export const getAllModelOptions = ( settings?: AiAssistantSettings, @@ -94,10 +101,10 @@ export const getAllModelOptions = ( const options: ModelOption[] = [] for (const providerId of Object.keys(BUILTIN_PROVIDERS)) { const enabledModels = settings.providers?.[providerId]?.enabledModels ?? [] - for (const modelId of enabledModels) { + for (const value of enabledModels) { options.push({ - label: getModelLabel(modelId, providerId, settings), - value: modelId, + label: getModelLabel(value, providerId, settings), + value, provider: providerId, }) } @@ -108,7 +115,7 @@ export const getAllModelOptions = ( for (const modelId of def.models) { options.push({ label: modelId, - value: makeCustomModelValue(providerId, modelId), + value: makeModelValue(providerId, modelId), provider: providerId, }) } @@ -120,14 +127,8 @@ export const providerForModel = ( model: ModelOption["value"], settings?: AiAssistantSettings, ): ProviderId | null => { - // Check for namespaced custom model value (providerId:modelId) const parsed = parseModelValue(model, settings?.customProviders) - if ("customProviderId" in parsed) return parsed.customProviderId - return ( - Object.keys(BUILTIN_PROVIDERS).find((providerId) => - settings?.providers?.[providerId]?.enabledModels?.includes(model), - ) ?? null - ) + return "providerId" in parsed ? parsed.providerId : null } export const getAllProviders = ( @@ -163,7 +164,7 @@ export const getAllEnabledModels = ( } else if (settings.customProviders?.[provider]) { models.push( ...settings.customProviders[provider].models.map((m) => - makeCustomModelValue(provider, m), + makeModelValue(provider, m), ), ) } @@ -175,25 +176,18 @@ export const getNextModel = ( currentModel: string | undefined, enabledModels: Record, settings?: AiAssistantSettings, - previousSettings?: AiAssistantSettings, ): string | null => { const providerOf = (model: string) => { - const parsed = parseModelValue( - model, - settings?.customProviders ?? previousSettings?.customProviders, - ) - if ("customProviderId" in parsed) return parsed.customProviderId - return ( - Object.keys(enabledModels).find((p) => - enabledModels[p]?.includes(model), - ) ?? - providerForModel(model, settings) ?? - providerForModel(model, previousSettings) - ) + const parsed = parseModelValue(model, settings?.customProviders) + return "providerId" in parsed ? parsed.providerId : null } const modelProvider = currentModel ? providerOf(currentModel) : null - if (modelProvider && enabledModels[modelProvider]?.length > 0) { - if (currentModel && enabledModels[modelProvider].includes(currentModel)) { + if ( + currentModel && + modelProvider && + enabledModels[modelProvider]?.length > 0 + ) { + if (enabledModels[modelProvider].includes(currentModel)) { return currentModel } return enabledModels[modelProvider][0] @@ -289,7 +283,9 @@ export const buildListingMetadata = ( } return { modelLabels, - ...(utilityModel ? { utilityModel } : {}), + ...(utilityModel + ? { utilityModel: makeModelValue(providerId, utilityModel) } + : {}), } } @@ -306,76 +302,6 @@ export const getProviderContextWindow = ( return custom?.contextWindow ?? null } -const LEGACY_REASONING_VARIANT = /@reasoning=(high|medium|low)$/ - -const collapseLegacyVariant = (modelId: string): string => - modelId.replace(LEGACY_REASONING_VARIANT, "") - -/** - * Reconciles persisted AI assistant settings. - * Collapses legacy `@reasoning=` model variants into plain ids and folds a - * selected high variant into the provider-level reasoningEffort. - * Validates custom provider models against customProviders definitions; - * built-in models stay until the Manage Models picker removes them. - * - * Pure function — does not write to localStorage. - * Idempotent: applying it multiple times produces the same result. - */ -export const reconcileSettings = ( - settings: AiAssistantSettings, -): AiAssistantSettings => { - const result = { - ...settings, - providers: { ...settings.providers }, - } - const selectedModel = result.selectedModel - - for (const providerKey of Object.keys(result.providers)) { - const providerSettings = result.providers[providerKey] - if (!providerSettings?.enabledModels) continue - - const isBuiltinProvider = Object.hasOwn(BUILTIN_PROVIDERS, providerKey) - const selectedHighVariant = - isBuiltinProvider && - selectedModel !== undefined && - selectedModel.endsWith("@reasoning=high") && - providerSettings.enabledModels.includes(selectedModel) - const validCustomIds = settings.customProviders?.[providerKey] - ? new Set( - settings.customProviders[providerKey].models.map((m) => - makeCustomModelValue(providerKey, m), - ), - ) - : null - result.providers[providerKey] = { - ...providerSettings, - enabledModels: isBuiltinProvider - ? [ - ...new Set( - providerSettings.enabledModels.map(collapseLegacyVariant), - ), - ] - : providerSettings.enabledModels.filter((id) => - validCustomIds?.has(id), - ), - ...(selectedHighVariant ? { reasoningEffort: "high" as const } : {}), - } - } - - if (result.selectedModel !== undefined) { - const selectedProvider = providerForModel(result.selectedModel, settings) - if ( - selectedProvider !== null && - Object.hasOwn(BUILTIN_PROVIDERS, selectedProvider) - ) { - result.selectedModel = collapseLegacyVariant(result.selectedModel) - } - } - result.selectedModel = getSelectedModel(result) ?? undefined - - return result -} - export const getApiKey = ( providerId: ProviderId, settings: AiAssistantSettings, @@ -422,17 +348,18 @@ export const getAiPermissions = ( } export const readLiveAiAssistantSettings = (): AiAssistantSettings | null => { - const stored = getValue(StoreKey.AI_ASSISTANT_SETTINGS) - if (!stored) return null try { + const stored = getValue(StoreKey.AI_ASSISTANT_SETTINGS) + if (!stored) return null const parsed = JSON.parse(stored) as AiAssistantSettings - return reconcileSettings({ + return { + modelValueFormat: parsed.modelValueFormat, selectedModel: parsed.selectedModel, providers: parsed.providers || {}, ...(parsed.customProviders && { customProviders: parsed.customProviders, }), - }) + } } catch { return null } diff --git a/src/utils/localStorage/migrate.ts b/src/utils/localStorage/migrate.ts new file mode 100644 index 000000000..4f1d00009 --- /dev/null +++ b/src/utils/localStorage/migrate.ts @@ -0,0 +1,163 @@ +import { + AI_MODEL_VALUE_FORMAT, + type AiAssistantSettings, + type CustomProviderDefinition, +} from "../../providers/LocalStorageProvider/types" +import { StoreKey } from "./types" + +type StorageAccess = Pick + +const BUILTIN_PROVIDER_IDS = ["anthropic", "openai"] as const +const LEGACY_REASONING_VARIANT = /@reasoning=(high|medium|low)$/ + +const makeModelValue = (providerId: string, modelId: string): string => + `${providerId}:${modelId}` + +const stripModelNamespace = (value: string, providerId: string): string => { + const prefix = `${providerId}:` + return value.startsWith(prefix) ? value.slice(prefix.length) : value +} + +const collapseLegacyVariant = (modelId: string): string => + modelId.replace(LEGACY_REASONING_VARIANT, "") + +const isBuiltinProvider = (providerId: string): boolean => + BUILTIN_PROVIDER_IDS.some((candidate) => candidate === providerId) + +const selectedProviderFromLegacySettings = ( + settings: AiAssistantSettings, +): { providerId: string; modelId: string } | null => { + const selectedModel = settings.selectedModel + if (!selectedModel) return null + + const separatorIndex = selectedModel.indexOf(":") + const prefix = + separatorIndex === -1 ? null : selectedModel.slice(0, separatorIndex) + if (prefix && Object.hasOwn(settings.customProviders ?? {}, prefix)) { + return { + providerId: prefix, + modelId: selectedModel.slice(separatorIndex + 1), + } + } + + const providerId = BUILTIN_PROVIDER_IDS.find((candidate) => + settings.providers[candidate]?.enabledModels?.includes(selectedModel), + ) + return providerId ? { providerId, modelId: selectedModel } : null +} + +const getEnabledModels = (settings: AiAssistantSettings): string[] => { + const providerIds = [ + ...BUILTIN_PROVIDER_IDS, + ...Object.keys(settings.customProviders ?? {}), + ] + return providerIds.flatMap((providerId) => { + const enabledModels = settings.providers[providerId]?.enabledModels + if (enabledModels) return enabledModels + return (settings.customProviders?.[providerId]?.models ?? []).map( + (modelId) => makeModelValue(providerId, modelId), + ) + }) +} + +/** Migrates the unversioned local-storage schema to the current schema. */ +export const migrateLocalStorage = ( + storage: StorageAccess = localStorage, +): boolean => { + try { + const stored = storage.getItem(StoreKey.AI_ASSISTANT_SETTINGS) + if (!stored) return true + + const settings = JSON.parse(stored) as AiAssistantSettings + + // Only the old, unversioned format is ours to migrate. Never reinterpret a + // current or future version. + if (settings.modelValueFormat !== undefined) return true + + const selected = selectedProviderFromLegacySettings(settings) + const migratedProviders: AiAssistantSettings["providers"] = { + ...settings.providers, + } + + for (const [providerId, providerSettings] of Object.entries( + settings.providers, + )) { + if (!providerSettings?.enabledModels) continue + + const builtin = isBuiltinProvider(providerId) + const customProvider: CustomProviderDefinition | undefined = + settings.customProviders?.[providerId] + const validCustomModels = customProvider + ? new Set(customProvider.models) + : null + const rawEnabledModels = providerSettings.enabledModels.map((value) => + builtin ? value : stripModelNamespace(value, providerId), + ) + const selectedHighVariant = + builtin && + selected?.providerId === providerId && + selected.modelId.endsWith("@reasoning=high") && + rawEnabledModels.includes(selected.modelId) + const enabledModels = [ + ...new Set( + rawEnabledModels + .map((modelId) => + builtin ? collapseLegacyVariant(modelId) : modelId, + ) + .filter((modelId) => builtin || validCustomModels?.has(modelId)) + .map((modelId) => makeModelValue(providerId, modelId)), + ), + ] + const rawUtilityModel = providerSettings.utilityModel + ? builtin + ? providerSettings.utilityModel + : stripModelNamespace(providerSettings.utilityModel, providerId) + : null + + migratedProviders[providerId] = { + ...providerSettings, + enabledModels, + ...(rawUtilityModel + ? { + utilityModel: makeModelValue( + providerId, + builtin + ? collapseLegacyVariant(rawUtilityModel) + : rawUtilityModel, + ), + } + : {}), + ...(selectedHighVariant ? { reasoningEffort: "high" as const } : {}), + } + } + + const migrated: AiAssistantSettings = { + ...settings, + modelValueFormat: AI_MODEL_VALUE_FORMAT, + providers: migratedProviders, + ...(selected + ? { + selectedModel: makeModelValue( + selected.providerId, + isBuiltinProvider(selected.providerId) + ? collapseLegacyVariant(selected.modelId) + : selected.modelId, + ), + } + : { selectedModel: undefined }), + } + const enabledModels = getEnabledModels(migrated) + if ( + !migrated.selectedModel || + !enabledModels.includes(migrated.selectedModel) + ) { + migrated.selectedModel = enabledModels[0] + } + + storage.setItem(StoreKey.AI_ASSISTANT_SETTINGS, JSON.stringify(migrated)) + return true + } catch { + // Leave invalid persisted data untouched; the regular reader uses defaults. + return false + } +} From a61cda50b4ae1ecf2de2bb5e0634b0a232f83331 Mon Sep 17 00:00:00 2001 From: emrberk Date: Sun, 6 Sep 2026 19:09:26 +0300 Subject: [PATCH 9/9] exemption for custom providers --- src/components/SetupAIAssistant/SettingsModal.tsx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/components/SetupAIAssistant/SettingsModal.tsx b/src/components/SetupAIAssistant/SettingsModal.tsx index 3e54075b5..812c20767 100644 --- a/src/components/SetupAIAssistant/SettingsModal.tsx +++ b/src/components/SetupAIAssistant/SettingsModal.tsx @@ -647,6 +647,17 @@ export const SettingsModal = ({ open, onOpenChange }: SettingsModalProps) => { if (isStale()) return const aiProvider = createProvider(provider, apiKey, localSettings) const classified = aiProvider.classifyError(err, () => {}) + if (!isBuiltin && classified.type !== "invalid_key") { + // Custom endpoints may not implement model listing or use standard + // HTTP statuses. Preserve the existing manual-configuration path. + setValidationState((prev) => ({ ...prev, [provider]: "validated" })) + setValidatedApiKeys((prev) => ({ ...prev, [provider]: true })) + setValidationErrors((prev) => ({ + ...prev, + [provider]: classified.message, + })) + return + } setValidationState((prev) => ({ ...prev, [provider]: "error" })) setValidatedApiKeys((prev) => ({ ...prev, [provider]: false })) setValidationErrors((prev) => ({