diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index fe869a5b77e..b8f8a650e65 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -3687,6 +3687,38 @@ export const SakanaIcon = (props: SVGProps) => ( ) +export const NvidiaIcon = (props: SVGProps) => ( + + NVIDIA + + +) + +export const ZaiIcon = (props: SVGProps) => ( + + Z.ai + + + + + +) + export function MetaIcon(props: SVGProps) { const id = useId() const gradient1Id = `meta_gradient_1_${id}` diff --git a/apps/sim/lib/api-key/byok.ts b/apps/sim/lib/api-key/byok.ts index 73df4a3a1b0..4a4f26b3271 100644 --- a/apps/sim/lib/api-key/byok.ts +++ b/apps/sim/lib/api-key/byok.ts @@ -204,13 +204,14 @@ export async function getApiKeyWithBYOK( const isClaudeModel = provider === 'anthropic' const isGeminiModel = provider === 'google' const isMistralModel = provider === 'mistral' + const isZaiModel = provider === 'zai' const byokProviderId = isGeminiModel ? 'google' : (provider as BYOKProviderId) if ( isHosted && workspaceId && - (isOpenAIModel || isClaudeModel || isGeminiModel || isMistralModel) + (isOpenAIModel || isClaudeModel || isGeminiModel || isMistralModel || isZaiModel) ) { const hostedModels = getHostedModels() const isModelHosted = hostedModels.some((m) => m.toLowerCase() === model.toLowerCase()) diff --git a/apps/sim/lib/api/contracts/byok-keys.ts b/apps/sim/lib/api/contracts/byok-keys.ts index 0c4f10f7085..d9b8a561099 100644 --- a/apps/sim/lib/api/contracts/byok-keys.ts +++ b/apps/sim/lib/api/contracts/byok-keys.ts @@ -6,6 +6,7 @@ export const byokProviderIdSchema = z.enum([ 'anthropic', 'google', 'mistral', + 'zai', 'fireworks', 'together', 'baseten', diff --git a/apps/sim/lib/core/config/api-keys.ts b/apps/sim/lib/core/config/api-keys.ts index ad093a4d987..1a1f04218cd 100644 --- a/apps/sim/lib/core/config/api-keys.ts +++ b/apps/sim/lib/core/config/api-keys.ts @@ -11,7 +11,8 @@ export function getRotatingApiKey(provider: string): string { provider !== 'openai' && provider !== 'anthropic' && provider !== 'gemini' && - provider !== 'cohere' + provider !== 'cohere' && + provider !== 'zai' ) { throw new Error(`No rotation implemented for provider: ${provider}`) } @@ -34,6 +35,10 @@ export function getRotatingApiKey(provider: string): string { if (env.COHERE_API_KEY_1) keys.push(env.COHERE_API_KEY_1) if (env.COHERE_API_KEY_2) keys.push(env.COHERE_API_KEY_2) if (env.COHERE_API_KEY_3) keys.push(env.COHERE_API_KEY_3) + } else if (provider === 'zai') { + if (env.ZAI_API_KEY_1) keys.push(env.ZAI_API_KEY_1) + if (env.ZAI_API_KEY_2) keys.push(env.ZAI_API_KEY_2) + if (env.ZAI_API_KEY_3) keys.push(env.ZAI_API_KEY_3) } if (keys.length === 0) { diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 990ab7679d3..6cf92e243a3 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -144,6 +144,9 @@ export const env = createEnv({ GEMINI_API_KEY_1: z.string().min(1).optional(), // Primary Gemini API key GEMINI_API_KEY_2: z.string().min(1).optional(), // Additional Gemini API key for load balancing GEMINI_API_KEY_3: z.string().min(1).optional(), // Additional Gemini API key for load balancing + ZAI_API_KEY_1: z.string().min(1).optional(), // Primary Z.ai API key for load balancing + ZAI_API_KEY_2: z.string().min(1).optional(), // Additional Z.ai API key for load balancing + ZAI_API_KEY_3: z.string().min(1).optional(), // Additional Z.ai API key for load balancing OLLAMA_URL: z.string().url().optional(), // Ollama local LLM server URL VLLM_BASE_URL: z.string().url().optional(), // vLLM self-hosted base URL (OpenAI-compatible) VLLM_API_KEY: z.string().optional(), // Optional bearer token for vLLM diff --git a/apps/sim/lib/tokenization/constants.ts b/apps/sim/lib/tokenization/constants.ts index 34d166ba2ed..7012e0dfe23 100644 --- a/apps/sim/lib/tokenization/constants.ts +++ b/apps/sim/lib/tokenization/constants.ts @@ -61,11 +61,21 @@ export const TOKENIZATION_CONFIG = { confidence: 'medium', supportedMethods: ['heuristic', 'fallback'], }, + nvidia: { + avgCharsPerToken: 4, + confidence: 'medium', + supportedMethods: ['heuristic', 'fallback'], + }, meta: { avgCharsPerToken: 4, confidence: 'medium', supportedMethods: ['heuristic', 'fallback'], }, + zai: { + avgCharsPerToken: 4, + confidence: 'medium', + supportedMethods: ['heuristic', 'fallback'], + }, ollama: { avgCharsPerToken: 4, confidence: 'low', diff --git a/apps/sim/providers/attachments.ts b/apps/sim/providers/attachments.ts index d6b37b2dce1..c902d959382 100644 --- a/apps/sim/providers/attachments.ts +++ b/apps/sim/providers/attachments.ts @@ -36,7 +36,9 @@ export type AttachmentProvider = | 'deepseek' | 'cerebras' | 'sakana' + | 'nvidia' | 'meta' + | 'zai' export interface PreparedProviderAttachment { file: UserFile @@ -124,7 +126,9 @@ const UNSUPPORTED_FILE_PROVIDERS = new Set([ 'deepseek', 'cerebras', 'sakana', + 'nvidia', 'meta', + 'zai', ]) const PROVIDER_SUPPORTED_LABELS: Record = { @@ -145,7 +149,9 @@ const PROVIDER_SUPPORTED_LABELS: Record = { deepseek: 'no file attachments in the current API adapter', cerebras: 'no file attachments in the current API adapter', sakana: 'no file attachments in the current API adapter', + nvidia: 'no file attachments in the current API adapter', meta: 'no file attachments in the current API adapter', + zai: 'no file attachments in the current API adapter', } export function getAttachmentProvider(providerId: ProviderId | string): AttachmentProvider | null { @@ -166,7 +172,9 @@ export function getAttachmentProvider(providerId: ProviderId | string): Attachme if (providerId === 'deepseek') return 'deepseek' if (providerId === 'cerebras') return 'cerebras' if (providerId === 'sakana') return 'sakana' + if (providerId === 'nvidia') return 'nvidia' if (providerId === 'meta') return 'meta' + if (providerId === 'zai') return 'zai' return null } @@ -315,7 +323,9 @@ function isMimeTypeSupportedByProvider( case 'deepseek': case 'cerebras': case 'sakana': + case 'nvidia': case 'meta': + case 'zai': return false default: { const _exhaustive: never = provider diff --git a/apps/sim/providers/models.test.ts b/apps/sim/providers/models.test.ts index b3b16b54bc7..0c01133649c 100644 --- a/apps/sim/providers/models.test.ts +++ b/apps/sim/providers/models.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it } from 'vitest' import { getBaseModelProviders, + getHostedModels, orderModelIdsByReleaseDate, PROVIDER_DEFINITIONS, } from '@/providers/models' @@ -134,3 +135,86 @@ describe('sakana provider definition', () => { expect(baseModels['fugu-ultra']).toBe('sakana') }) }) + +describe('nvidia provider definition', () => { + const nvidia = PROVIDER_DEFINITIONS.nvidia + + const expectedModels = [ + { id: 'nvidia/llama-3.1-nemotron-70b-instruct', contextWindow: 128000 }, + { id: 'nvidia/llama-3.1-nemotron-ultra-253b-v1', contextWindow: 131072 }, + { id: 'nvidia/llama-3.3-nemotron-super-49b-v1.5', contextWindow: 131072 }, + { id: 'nvidia/nemotron-3-nano-30b-a3b', contextWindow: 262144 }, + { id: 'nvidia/nemotron-3-super-120b-a12b', contextWindow: 1048576 }, + { id: 'nvidia/nemotron-3-ultra-550b-a55b', contextWindow: 1048576 }, + ] + + it('is registered with the current-gen Super model as the default', () => { + expect(nvidia).toBeDefined() + expect(nvidia.id).toBe('nvidia') + expect(nvidia.defaultModel).toBe('nvidia/nemotron-3-super-120b-a12b') + expect(nvidia.modelPatterns).toEqual([/^nvidia\//]) + }) + + it('exposes all six Nemotron models with the documented context windows', () => { + expect(nvidia.models.map((m) => m.id)).toEqual(expectedModels.map((m) => m.id)) + for (const expected of expectedModels) { + const model = nvidia.models.find((m) => m.id === expected.id) + expect(model?.contextWindow).toBe(expected.contextWindow) + } + }) + + it('routes every nvidia model ID to the nvidia provider', () => { + const baseModels = getBaseModelProviders() + for (const expected of expectedModels) { + expect(baseModels[expected.id]).toBe('nvidia') + } + }) +}) + +describe('zai provider definition', () => { + const zai = PROVIDER_DEFINITIONS.zai + + const expectedModels = [ + { id: 'glm-5.2', contextWindow: 1000000 }, + { id: 'glm-5.1', contextWindow: 200000 }, + { id: 'glm-5', contextWindow: 200000 }, + { id: 'glm-5-turbo', contextWindow: 200000 }, + { id: 'glm-4.7', contextWindow: 200000 }, + { id: 'glm-4.7-flashx', contextWindow: 200000 }, + { id: 'glm-4.6', contextWindow: 200000 }, + { id: 'glm-4.5', contextWindow: 128000 }, + { id: 'glm-4.5-air', contextWindow: 128000 }, + { id: 'glm-4.5-x', contextWindow: 128000 }, + { id: 'glm-4.5-airx', contextWindow: 128000 }, + { id: 'glm-4-32b-0414-128k', contextWindow: 128000 }, + ] + + it('is registered with a bare glm-4.6 as the default model', () => { + expect(zai).toBeDefined() + expect(zai.id).toBe('zai') + expect(zai.defaultModel).toBe('glm-4.6') + expect(zai.defaultModel.startsWith('zai/')).toBe(false) + // No fallback pattern — an unscoped `/^glm/` would overmatch unrelated self-hosted + // "glm-*" models and misroute them to Z.ai's hosted billing. + expect(zai.modelPatterns).toEqual([]) + }) + + it('exposes every GLM model with the documented context window', () => { + expect(zai.models.map((m) => m.id)).toEqual(expectedModels.map((m) => m.id)) + for (const expected of expectedModels) { + const model = zai.models.find((m) => m.id === expected.id) + expect(model?.contextWindow).toBe(expected.contextWindow) + } + }) + + it('routes every bare glm-* model ID to the zai provider', () => { + const baseModels = getBaseModelProviders() + for (const expected of expectedModels) { + expect(baseModels[expected.id]).toBe('zai') + } + }) + + it('is included in getHostedModels since Sim provides the Z.ai key server-side', () => { + expect(getHostedModels()).toContain('glm-4.6') + }) +}) diff --git a/apps/sim/providers/models.ts b/apps/sim/providers/models.ts index 830492aeb2f..b9245ee4106 100644 --- a/apps/sim/providers/models.ts +++ b/apps/sim/providers/models.ts @@ -21,6 +21,7 @@ import { LitellmIcon, MetaIcon, MistralIcon, + NvidiaIcon, OllamaIcon, OpenAIIcon, OpenRouterIcon, @@ -29,6 +30,7 @@ import { VertexIcon, VllmIcon, xAIIcon, + ZaiIcon, } from '@/components/icons' import type { ModelPricing, ProviderId } from '@/providers/types' @@ -2353,6 +2355,115 @@ export const PROVIDER_DEFINITIONS: Record = { }, ], }, + nvidia: { + id: 'nvidia', + name: 'NVIDIA NIM', + description: "NVIDIA's Nemotron models via NIM's OpenAI-compatible API", + defaultModel: 'nvidia/nemotron-3-super-120b-a12b', + modelPatterns: [/^nvidia\//], + icon: NvidiaIcon, + color: '#77B900', + isReseller: true, + contextInformationAvailable: true, + capabilities: { + temperature: { min: 0, max: 2 }, + toolUsageControl: true, + }, + models: [ + { + id: 'nvidia/llama-3.1-nemotron-70b-instruct', + pricing: { + input: 1.2, + output: 1.2, + updatedAt: '2026-07-09', + }, + capabilities: { + temperature: { min: 0, max: 2 }, + toolUsageControl: true, + maxOutputTokens: 4096, + }, + contextWindow: 128000, + releaseDate: '2024-10-15', + }, + { + id: 'nvidia/llama-3.1-nemotron-ultra-253b-v1', + pricing: { + input: 0.6, + output: 1.8, + updatedAt: '2026-07-09', + }, + capabilities: { + temperature: { min: 0, max: 2 }, + toolUsageControl: true, + maxOutputTokens: 8192, + }, + contextWindow: 131072, + releaseDate: '2025-04-07', + }, + { + id: 'nvidia/llama-3.3-nemotron-super-49b-v1.5', + pricing: { + input: 0.4, + output: 0.4, + updatedAt: '2026-07-09', + }, + capabilities: { + temperature: { min: 0, max: 2 }, + toolUsageControl: true, + maxOutputTokens: 8192, + }, + contextWindow: 131072, + releaseDate: '2025-07-25', + }, + { + id: 'nvidia/nemotron-3-nano-30b-a3b', + pricing: { + input: 0.05, + output: 0.2, + updatedAt: '2026-07-09', + }, + capabilities: { + temperature: { min: 0, max: 2 }, + toolUsageControl: true, + maxOutputTokens: 8192, + }, + contextWindow: 262144, + releaseDate: '2025-12-15', + speedOptimized: true, + }, + { + id: 'nvidia/nemotron-3-super-120b-a12b', + pricing: { + input: 0.08, + output: 0.45, + updatedAt: '2026-07-09', + }, + capabilities: { + temperature: { min: 0, max: 2 }, + toolUsageControl: true, + maxOutputTokens: 8192, + }, + contextWindow: 1048576, + releaseDate: '2026-03-11', + recommended: true, + }, + { + id: 'nvidia/nemotron-3-ultra-550b-a55b', + pricing: { + input: 0.5, + output: 2.2, + updatedAt: '2026-07-09', + }, + capabilities: { + temperature: { min: 0, max: 2 }, + toolUsageControl: true, + maxOutputTokens: 8192, + }, + contextWindow: 1048576, + releaseDate: '2026-06-04', + }, + ], + }, meta: { id: 'meta', name: 'Meta', @@ -2385,6 +2496,237 @@ export const PROVIDER_DEFINITIONS: Record = { }, ], }, + zai: { + id: 'zai', + name: 'Z.ai', + description: "Z.ai's GLM models via an OpenAI-compatible API", + defaultModel: 'glm-4.6', + // No fallback pattern: `/^glm/` would overmatch any unrelated self-hosted "glm-*" model + // (e.g. a custom vLLM/LiteLLM deployment) and misroute it to Z.ai's hosted billing. + // Routing for zai's own models relies solely on the exact catalog match in `models`. + modelPatterns: [], + icon: ZaiIcon, + color: '#2D2D2D', + contextInformationAvailable: true, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + }, + models: [ + { + id: 'glm-5.2', + pricing: { + input: 1.4, + output: 4.4, + updatedAt: '2026-07-10', + }, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + maxOutputTokens: 131072, + reasoningEffort: { + values: ['none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'], + }, + }, + contextWindow: 1000000, + releaseDate: '2026-06-13', + recommended: true, + }, + { + id: 'glm-5.1', + pricing: { + input: 1.4, + output: 4.4, + updatedAt: '2026-07-10', + }, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + maxOutputTokens: 131072, + thinking: { + levels: ['disabled', 'enabled'], + default: 'enabled', + }, + }, + contextWindow: 200000, + releaseDate: '2026-04-07', + }, + { + id: 'glm-5', + pricing: { + input: 1.0, + output: 3.2, + updatedAt: '2026-07-10', + }, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + maxOutputTokens: 131072, + thinking: { + levels: ['disabled', 'enabled'], + default: 'enabled', + }, + }, + contextWindow: 200000, + releaseDate: '2026-02-11', + }, + { + id: 'glm-5-turbo', + pricing: { + input: 1.2, + output: 4.0, + updatedAt: '2026-07-10', + }, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + maxOutputTokens: 131072, + thinking: { + levels: ['disabled', 'enabled'], + default: 'enabled', + }, + }, + contextWindow: 200000, + releaseDate: '2026-03-15', + }, + { + id: 'glm-4.7', + pricing: { + input: 0.6, + output: 2.2, + updatedAt: '2026-07-10', + }, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + maxOutputTokens: 131072, + thinking: { + levels: ['disabled', 'enabled'], + default: 'enabled', + }, + }, + contextWindow: 200000, + releaseDate: '2025-12-22', + }, + { + id: 'glm-4.7-flashx', + pricing: { + input: 0.07, + output: 0.4, + updatedAt: '2026-07-10', + }, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + maxOutputTokens: 131072, + }, + contextWindow: 200000, + releaseDate: '2025-12-22', + }, + { + id: 'glm-4.6', + pricing: { + input: 0.6, + output: 2.2, + updatedAt: '2026-07-10', + }, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + maxOutputTokens: 131072, + thinking: { + levels: ['disabled', 'enabled'], + default: 'enabled', + }, + }, + contextWindow: 200000, + releaseDate: '2025-09-30', + }, + { + id: 'glm-4.5', + pricing: { + input: 0.6, + output: 2.2, + updatedAt: '2026-07-10', + }, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + maxOutputTokens: 98304, + thinking: { + levels: ['disabled', 'enabled'], + default: 'enabled', + }, + }, + contextWindow: 128000, + releaseDate: '2025-07-28', + }, + { + id: 'glm-4.5-air', + pricing: { + input: 0.2, + output: 1.1, + updatedAt: '2026-07-10', + }, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + maxOutputTokens: 98304, + thinking: { + levels: ['disabled', 'enabled'], + default: 'enabled', + }, + }, + contextWindow: 128000, + releaseDate: '2025-07-28', + }, + { + id: 'glm-4.5-x', + pricing: { + input: 2.2, + output: 8.9, + updatedAt: '2026-07-10', + }, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + maxOutputTokens: 98304, + }, + contextWindow: 128000, + releaseDate: '2025-07-28', + }, + { + id: 'glm-4.5-airx', + pricing: { + input: 1.1, + output: 4.5, + updatedAt: '2026-07-10', + }, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + maxOutputTokens: 98304, + }, + contextWindow: 128000, + releaseDate: '2025-07-28', + }, + { + id: 'glm-4-32b-0414-128k', + pricing: { + input: 0.1, + output: 0.1, + updatedAt: '2026-07-10', + }, + capabilities: { + temperature: { min: 0, max: 1 }, + toolUsageControl: true, + maxOutputTokens: 16384, + }, + contextWindow: 128000, + releaseDate: '2025-04-14', + }, + ], + }, mistral: { id: 'mistral', name: 'Mistral AI', @@ -3528,6 +3870,7 @@ export function getHostedModels(): string[] { ...getProviderModels('openai'), ...getProviderModels('anthropic'), ...getProviderModels('google'), + ...getProviderModels('zai'), ] } diff --git a/apps/sim/providers/nvidia/index.ts b/apps/sim/providers/nvidia/index.ts new file mode 100644 index 00000000000..0b147d29b10 --- /dev/null +++ b/apps/sim/providers/nvidia/index.ts @@ -0,0 +1,632 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import OpenAI from 'openai' +import type { StreamingExecution } from '@/executor/types' +import { MAX_TOOL_ITERATIONS } from '@/providers' +import { formatMessagesForProvider } from '@/providers/attachments' +import { getProviderDefaultModel, getProviderModels } from '@/providers/models' +import { createReadableStreamFromNvidiaStream } from '@/providers/nvidia/utils' +import { createStreamingExecution } from '@/providers/streaming-execution' +import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter' +import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment' +import type { + ProviderConfig, + ProviderRequest, + ProviderResponse, + TimeSegment, +} from '@/providers/types' +import { ProviderError } from '@/providers/types' +import { + calculateCost, + prepareToolExecution, + prepareToolsWithUsageControl, + sumToolCosts, + trackForcedToolUsage, +} from '@/providers/utils' +import { executeTool } from '@/tools' + +const logger = createLogger('NvidiaProvider') + +const NVIDIA_BASE_URL = 'https://integrate.api.nvidia.com/v1' + +export const nvidiaProvider: ProviderConfig = { + id: 'nvidia', + name: 'NVIDIA NIM', + description: "NVIDIA's Nemotron models via NIM's OpenAI-compatible API", + version: '1.0.0', + models: getProviderModels('nvidia'), + defaultModel: getProviderDefaultModel('nvidia'), + + executeRequest: async ( + request: ProviderRequest + ): Promise => { + if (!request.apiKey) { + throw new Error('API key is required for NVIDIA NIM') + } + + const providerStartTime = Date.now() + const providerStartTimeISO = new Date(providerStartTime).toISOString() + + try { + const nvidia = new OpenAI({ + apiKey: request.apiKey, + baseURL: NVIDIA_BASE_URL, + }) + + const allMessages = [] + + if (request.systemPrompt) { + allMessages.push({ + role: 'system', + content: request.systemPrompt, + }) + } + + if (request.context) { + allMessages.push({ + role: 'user', + content: request.context, + }) + } + + if (request.messages) { + allMessages.push(...request.messages) + } + const formattedMessages = formatMessagesForProvider(allMessages, 'nvidia') + + const tools = request.tools?.length + ? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool)) + : undefined + + const payload: any = { + model: request.model, + messages: formattedMessages, + } + + if (request.temperature !== undefined) payload.temperature = request.temperature + if (request.maxTokens != null) payload.max_completion_tokens = request.maxTokens + + const responseFormatPayload = request.responseFormat + ? { + type: 'json_schema' as const, + json_schema: { + name: request.responseFormat.name || 'response_schema', + schema: request.responseFormat.schema || request.responseFormat, + strict: request.responseFormat.strict !== false, + }, + } + : undefined + + let preparedTools: ReturnType | null = null + let hasActiveTools = false + + if (tools?.length) { + preparedTools = prepareToolsWithUsageControl(tools, request.tools, logger, 'openai') + const { tools: filteredTools, toolChoice } = preparedTools + + if (filteredTools?.length && toolChoice) { + payload.tools = filteredTools + payload.tool_choice = toolChoice + hasActiveTools = true + + logger.info('NVIDIA NIM request configuration:', { + toolCount: filteredTools.length, + toolChoice: + typeof toolChoice === 'string' + ? toolChoice + : toolChoice.type === 'function' + ? `force:${toolChoice.function.name}` + : 'unknown', + model: request.model, + }) + } + } + + // Structured output and tool calling cannot be sent together — OpenAI-compatible + // backends reject a request that carries both `response_format` and active + // `tools`/`tool_choice`. Defer the schema until after the tool loop completes. + const deferResponseFormat = !!responseFormatPayload && hasActiveTools + if (responseFormatPayload && !deferResponseFormat) { + payload.response_format = responseFormatPayload + } + + if (request.stream && (!tools || tools.length === 0 || !hasActiveTools)) { + logger.info('Using streaming response for NVIDIA NIM request (no tools)') + + const streamResponse = await nvidia.chat.completions.create( + { + ...payload, + stream: true, + stream_options: { include_usage: true }, + }, + request.abortSignal ? { signal: request.abortSignal } : undefined + ) + + const streamingResult = createStreamingExecution({ + model: request.model, + providerStartTime, + providerStartTimeISO, + timing: { kind: 'simple', segmentName: request.model }, + initialTokens: { input: 0, output: 0, total: 0 }, + initialCost: { input: 0, output: 0, total: 0 }, + isStreaming: true, + createStream: ({ output }) => + createReadableStreamFromNvidiaStream(streamResponse as any, (content, usage) => { + output.content = content + output.tokens = { + input: usage.prompt_tokens, + output: usage.completion_tokens, + total: usage.total_tokens, + } + + const costResult = calculateCost( + request.model, + usage.prompt_tokens, + usage.completion_tokens + ) + output.cost = { + input: costResult.input, + output: costResult.output, + total: costResult.total, + } + }), + }) + + return streamingResult + } + + const initialCallTime = Date.now() + const originalToolChoice = payload.tool_choice + const forcedTools = preparedTools?.forcedTools || [] + let usedForcedTools: string[] = [] + + let currentResponse = await nvidia.chat.completions.create( + payload, + request.abortSignal ? { signal: request.abortSignal } : undefined + ) + const firstResponseTime = Date.now() - initialCallTime + + let content = currentResponse.choices[0]?.message?.content || '' + + const tokens = { + input: currentResponse.usage?.prompt_tokens || 0, + output: currentResponse.usage?.completion_tokens || 0, + total: currentResponse.usage?.total_tokens || 0, + } + const toolCalls = [] + const toolResults: Record[] = [] + const currentMessages = [...formattedMessages] + let iterationCount = 0 + let hasUsedForcedTool = false + let modelTime = firstResponseTime + let toolsTime = 0 + + const timeSegments: TimeSegment[] = [ + { + type: 'model', + name: request.model, + startTime: initialCallTime, + endTime: initialCallTime + firstResponseTime, + duration: firstResponseTime, + }, + ] + + if ( + typeof originalToolChoice === 'object' && + currentResponse.choices[0]?.message?.tool_calls + ) { + const toolCallsResponse = currentResponse.choices[0].message.tool_calls + const result = trackForcedToolUsage( + toolCallsResponse, + originalToolChoice, + logger, + 'openai', + forcedTools, + usedForcedTools + ) + hasUsedForcedTool = result.hasUsedForcedTool + usedForcedTools = result.usedForcedTools + } + + try { + while (iterationCount < MAX_TOOL_ITERATIONS) { + if (currentResponse.choices[0]?.message?.content) { + content = currentResponse.choices[0].message.content + } + + const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + + enrichLastModelSegmentFromChatCompletions( + timeSegments, + currentResponse, + toolCallsInResponse, + { model: request.model, provider: 'nvidia' } + ) + + if (!toolCallsInResponse || toolCallsInResponse.length === 0) { + break + } + + const toolsStartTime = Date.now() + + const toolExecutionPromises = toolCallsInResponse.map(async (toolCall) => { + const toolCallStartTime = Date.now() + const toolName = toolCall.function.name + + try { + const toolArgs = JSON.parse(toolCall.function.arguments) + const tool = request.tools?.find((t) => t.id === toolName) + + // Every tool_call in the assistant message must be answered by a matching + // `tool` message, or the next request violates the OpenAI message contract. + // Emit an error result for an unknown tool rather than dropping it. + if (!tool) { + const toolCallEndTime = Date.now() + return { + toolCall, + toolName, + toolParams: {}, + result: { + success: false, + output: undefined, + error: `Tool "${toolName}" is not available`, + }, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallEndTime - toolCallStartTime, + } + } + + const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) + const result = await executeTool(toolName, executionParams, { + signal: request.abortSignal, + }) + const toolCallEndTime = Date.now() + + return { + toolCall, + toolName, + toolParams, + result, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallEndTime - toolCallStartTime, + } + } catch (error) { + const toolCallEndTime = Date.now() + logger.error('Error processing tool call:', { error, toolName }) + + return { + toolCall, + toolName, + toolParams: {}, + result: { + success: false, + output: undefined, + error: getErrorMessage(error, 'Tool execution failed'), + }, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallEndTime - toolCallStartTime, + } + } + }) + + const executionResults = await Promise.allSettled(toolExecutionPromises) + + currentMessages.push({ + role: 'assistant', + content: null, + tool_calls: toolCallsInResponse.map((tc) => ({ + id: tc.id, + type: 'function', + function: { + name: tc.function.name, + arguments: tc.function.arguments, + }, + })), + }) + + for (const settledResult of executionResults) { + if (settledResult.status === 'rejected' || !settledResult.value) continue + + const { toolCall, toolName, toolParams, result, startTime, endTime, duration } = + settledResult.value + + timeSegments.push({ + type: 'tool', + name: toolName, + startTime: startTime, + endTime: endTime, + duration: duration, + toolCallId: toolCall.id, + }) + + let resultContent: any + if (result.success && result.output) { + toolResults.push(result.output) + resultContent = result.output + } else { + resultContent = { + error: true, + message: result.error || 'Tool execution failed', + tool: toolName, + } + } + + toolCalls.push({ + name: toolName, + arguments: toolParams, + startTime: new Date(startTime).toISOString(), + endTime: new Date(endTime).toISOString(), + duration: duration, + result: resultContent, + success: result.success, + }) + + currentMessages.push({ + role: 'tool', + tool_call_id: toolCall.id, + content: JSON.stringify(resultContent), + }) + } + + const thisToolsTime = Date.now() - toolsStartTime + toolsTime += thisToolsTime + + const nextPayload = { + ...payload, + messages: currentMessages, + } + + if ( + typeof originalToolChoice === 'object' && + hasUsedForcedTool && + forcedTools.length > 0 + ) { + const remainingTools = forcedTools.filter((tool) => !usedForcedTools.includes(tool)) + + if (remainingTools.length > 0) { + nextPayload.tool_choice = { + type: 'function', + function: { name: remainingTools[0] }, + } + logger.info(`Forcing next tool: ${remainingTools[0]}`) + } else { + nextPayload.tool_choice = 'auto' + logger.info('All forced tools have been used, switching to auto tool_choice') + } + } + + const nextModelStartTime = Date.now() + currentResponse = await nvidia.chat.completions.create( + nextPayload, + request.abortSignal ? { signal: request.abortSignal } : undefined + ) + + if ( + typeof nextPayload.tool_choice === 'object' && + currentResponse.choices[0]?.message?.tool_calls + ) { + const toolCallsResponse = currentResponse.choices[0].message.tool_calls + const result = trackForcedToolUsage( + toolCallsResponse, + nextPayload.tool_choice, + logger, + 'openai', + forcedTools, + usedForcedTools + ) + hasUsedForcedTool = result.hasUsedForcedTool + usedForcedTools = result.usedForcedTools + } + + const nextModelEndTime = Date.now() + const thisModelTime = nextModelEndTime - nextModelStartTime + + timeSegments.push({ + type: 'model', + name: request.model, + startTime: nextModelStartTime, + endTime: nextModelEndTime, + duration: thisModelTime, + }) + + modelTime += thisModelTime + + if (currentResponse.choices[0]?.message?.content) { + content = currentResponse.choices[0].message.content + } + + if (currentResponse.usage) { + tokens.input += currentResponse.usage.prompt_tokens || 0 + tokens.output += currentResponse.usage.completion_tokens || 0 + tokens.total += currentResponse.usage.total_tokens || 0 + } + + iterationCount++ + } + + if (iterationCount === MAX_TOOL_ITERATIONS) { + enrichLastModelSegmentFromChatCompletions( + timeSegments, + currentResponse, + currentResponse.choices[0]?.message?.tool_calls, + { model: request.model, provider: 'nvidia' } + ) + } + } catch (error) { + logger.error('Error in NVIDIA NIM request:', { error }) + throw error + } + + if (request.stream) { + logger.info('Using streaming for final NVIDIA NIM response after tool processing') + + // The tool loop is complete: this final pass only produces the textual answer. + // Force `tool_choice: 'none'` so the model cannot emit fresh tool calls that the + // text-only stream adapter would silently drop. + const streamingPayload: any = { + ...payload, + messages: currentMessages, + tool_choice: 'none', + stream: true, + stream_options: { include_usage: true }, + } + if (deferResponseFormat && responseFormatPayload) { + streamingPayload.response_format = responseFormatPayload + streamingPayload.parallel_tool_calls = false + } + + const streamResponse = await nvidia.chat.completions.create( + streamingPayload, + request.abortSignal ? { signal: request.abortSignal } : undefined + ) + + const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output) + + const streamingResult = createStreamingExecution({ + model: request.model, + providerStartTime, + providerStartTimeISO, + timing: { + kind: 'accumulated', + modelTime, + toolsTime, + firstResponseTime, + iterations: iterationCount + 1, + timeSegments, + }, + initialTokens: { + input: tokens.input, + output: tokens.output, + total: tokens.total, + }, + initialCost: { + input: accumulatedCost.input, + output: accumulatedCost.output, + toolCost: undefined as number | undefined, + total: accumulatedCost.total, + }, + toolCalls: + toolCalls.length > 0 + ? { + list: toolCalls, + count: toolCalls.length, + } + : undefined, + isStreaming: true, + createStream: ({ output }) => + createReadableStreamFromNvidiaStream(streamResponse as any, (content, usage) => { + output.content = content + output.tokens = { + input: tokens.input + usage.prompt_tokens, + output: tokens.output + usage.completion_tokens, + total: tokens.total + usage.total_tokens, + } + + const streamCost = calculateCost( + request.model, + usage.prompt_tokens, + usage.completion_tokens + ) + const tc = sumToolCosts(toolResults) + output.cost = { + input: accumulatedCost.input + streamCost.input, + output: accumulatedCost.output + streamCost.output, + toolCost: tc || undefined, + total: accumulatedCost.total + streamCost.total + tc, + } + }), + }) + + return streamingResult + } + + // Tools were active, so `response_format` was withheld from the loop. Make one final + // tool-free call to obtain the structured response now that the tool work is done. + if (deferResponseFormat && responseFormatPayload) { + logger.info('Applying deferred JSON schema response format after tool processing') + + const finalFormatStartTime = Date.now() + const finalPayload: any = { + ...payload, + messages: currentMessages, + response_format: responseFormatPayload, + tool_choice: 'none', + parallel_tool_calls: false, + } + + currentResponse = await nvidia.chat.completions.create( + finalPayload, + request.abortSignal ? { signal: request.abortSignal } : undefined + ) + + const finalFormatEndTime = Date.now() + timeSegments.push({ + type: 'model', + name: request.model, + startTime: finalFormatStartTime, + endTime: finalFormatEndTime, + duration: finalFormatEndTime - finalFormatStartTime, + }) + modelTime += finalFormatEndTime - finalFormatStartTime + + const formattedContent = currentResponse.choices[0]?.message?.content + if (formattedContent) { + content = formattedContent + } + + if (currentResponse.usage) { + tokens.input += currentResponse.usage.prompt_tokens || 0 + tokens.output += currentResponse.usage.completion_tokens || 0 + tokens.total += currentResponse.usage.total_tokens || 0 + } + + enrichLastModelSegmentFromChatCompletions( + timeSegments, + currentResponse, + currentResponse.choices[0]?.message?.tool_calls, + { model: request.model, provider: 'nvidia' } + ) + } + + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime + + return { + content, + model: request.model, + tokens, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + toolResults: toolResults.length > 0 ? toolResults : undefined, + timing: { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + modelTime: modelTime, + toolsTime: toolsTime, + firstResponseTime: firstResponseTime, + iterations: iterationCount + 1, + timeSegments: timeSegments, + }, + } + } catch (error) { + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime + + logger.error('Error in NVIDIA NIM request:', { + error, + duration: totalDuration, + }) + + throw new ProviderError(toError(error).message, { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + }) + } + }, +} diff --git a/apps/sim/providers/nvidia/utils.ts b/apps/sim/providers/nvidia/utils.ts new file mode 100644 index 00000000000..ef9c37b3a50 --- /dev/null +++ b/apps/sim/providers/nvidia/utils.ts @@ -0,0 +1,14 @@ +import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import type { CompletionUsage } from 'openai/resources/completions' +import { createOpenAICompatibleStream } from '@/providers/utils' + +/** + * Creates a ReadableStream from an NVIDIA NIM streaming response. + * Uses the shared OpenAI-compatible streaming utility. + */ +export function createReadableStreamFromNvidiaStream( + nvidiaStream: AsyncIterable, + onComplete?: (content: string, usage: CompletionUsage) => void +): ReadableStream { + return createOpenAICompatibleStream(nvidiaStream, 'NVIDIA', onComplete) +} diff --git a/apps/sim/providers/registry.ts b/apps/sim/providers/registry.ts index a221966914c..9e98d02d134 100644 --- a/apps/sim/providers/registry.ts +++ b/apps/sim/providers/registry.ts @@ -13,6 +13,7 @@ import { groqProvider } from '@/providers/groq' import { litellmProvider } from '@/providers/litellm' import { metaProvider } from '@/providers/meta' import { mistralProvider } from '@/providers/mistral' +import { nvidiaProvider } from '@/providers/nvidia' import { ollamaProvider } from '@/providers/ollama' import { ollamaCloudProvider } from '@/providers/ollama-cloud' import { openaiProvider } from '@/providers/openai' @@ -23,6 +24,7 @@ import type { ProviderConfig, ProviderId } from '@/providers/types' import { vertexProvider } from '@/providers/vertex' import { vllmProvider } from '@/providers/vllm' import { xAIProvider } from '@/providers/xai' +import { zaiProvider } from '@/providers/zai' const logger = createLogger('ProviderRegistry') @@ -37,7 +39,9 @@ const providerRegistry: Record = { cerebras: cerebrasProvider, groq: groqProvider, sakana: sakanaProvider, + nvidia: nvidiaProvider, meta: metaProvider, + zai: zaiProvider, vllm: vllmProvider, litellm: litellmProvider, mistral: mistralProvider, diff --git a/apps/sim/providers/types.ts b/apps/sim/providers/types.ts index 2cb08a6f6c6..01752a9c1b8 100644 --- a/apps/sim/providers/types.ts +++ b/apps/sim/providers/types.ts @@ -12,7 +12,9 @@ export type ProviderId = | 'cerebras' | 'groq' | 'sakana' + | 'nvidia' | 'meta' + | 'zai' | 'mistral' | 'ollama' | 'ollama-cloud' diff --git a/apps/sim/providers/utils.ts b/apps/sim/providers/utils.ts index b9848bc5d73..3b8bdd10c6a 100644 --- a/apps/sim/providers/utils.ts +++ b/apps/sim/providers/utils.ts @@ -155,7 +155,9 @@ export const providers: Record = { cerebras: buildProviderMetadata('cerebras'), groq: buildProviderMetadata('groq'), sakana: buildProviderMetadata('sakana'), + nvidia: buildProviderMetadata('nvidia'), meta: buildProviderMetadata('meta'), + zai: buildProviderMetadata('zai'), mistral: buildProviderMetadata('mistral'), bedrock: buildProviderMetadata('bedrock'), openrouter: buildProviderMetadata('openrouter'), @@ -900,8 +902,9 @@ export function getApiKey(provider: string, model: string, userProvidedKey?: str const isOpenAIModel = provider === 'openai' const isClaudeModel = provider === 'anthropic' const isGeminiModel = provider === 'google' + const isZaiModel = provider === 'zai' - if (isHosted && (isOpenAIModel || isClaudeModel || isGeminiModel)) { + if (isHosted && (isOpenAIModel || isClaudeModel || isGeminiModel || isZaiModel)) { const hostedModels = getHostedModels() const isModelHosted = hostedModels.some((m) => m.toLowerCase() === model.toLowerCase()) diff --git a/apps/sim/providers/zai/index.ts b/apps/sim/providers/zai/index.ts new file mode 100644 index 00000000000..d254546d6eb --- /dev/null +++ b/apps/sim/providers/zai/index.ts @@ -0,0 +1,649 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import OpenAI from 'openai' +import type { StreamingExecution } from '@/executor/types' +import { MAX_TOOL_ITERATIONS } from '@/providers' +import { formatMessagesForProvider } from '@/providers/attachments' +import { getProviderDefaultModel, getProviderModels } from '@/providers/models' +import { createStreamingExecution } from '@/providers/streaming-execution' +import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter' +import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment' +import type { + ProviderConfig, + ProviderRequest, + ProviderResponse, + TimeSegment, +} from '@/providers/types' +import { ProviderError } from '@/providers/types' +import { + calculateCost, + prepareToolExecution, + prepareToolsWithUsageControl, + sumToolCosts, + trackForcedToolUsage, +} from '@/providers/utils' +import { createReadableStreamFromZaiStream } from '@/providers/zai/utils' +import { executeTool } from '@/tools' + +const logger = createLogger('ZaiProvider') + +const ZAI_BASE_URL = 'https://api.z.ai/api/paas/v4' + +export const zaiProvider: ProviderConfig = { + id: 'zai', + name: 'Z.ai', + description: "Z.ai's GLM models via an OpenAI-compatible API", + version: '1.0.0', + models: getProviderModels('zai'), + defaultModel: getProviderDefaultModel('zai'), + + executeRequest: async ( + request: ProviderRequest + ): Promise => { + if (!request.apiKey) { + throw new Error('API key is required for Z.ai') + } + + const providerStartTime = Date.now() + const providerStartTimeISO = new Date(providerStartTime).toISOString() + + try { + const zai = new OpenAI({ + apiKey: request.apiKey, + baseURL: ZAI_BASE_URL, + }) + + const allMessages = [] + + if (request.systemPrompt) { + allMessages.push({ + role: 'system', + content: request.systemPrompt, + }) + } + + if (request.context) { + allMessages.push({ + role: 'user', + content: request.context, + }) + } + + if (request.messages) { + allMessages.push(...request.messages) + } + const formattedMessages = formatMessagesForProvider(allMessages, 'zai') + + const tools = request.tools?.length + ? request.tools.map((tool) => adaptOpenAIChatToolSchema(tool)) + : undefined + + const payload: any = { + model: request.model, + messages: formattedMessages, + } + + if (request.temperature !== undefined) payload.temperature = request.temperature + if (request.maxTokens != null) payload.max_completion_tokens = request.maxTokens + + // GLM's `thinking` toggle (models where `capabilities.thinking.levels` is + // ['disabled', 'enabled']) maps directly to Z.ai's `thinking: { type }` request param. + if (request.thinkingLevel === 'enabled' || request.thinkingLevel === 'disabled') { + payload.thinking = { type: request.thinkingLevel } + } + + // GLM-5.2's `reasoningEffort` capability maps directly to Z.ai's `reasoning_effort` + // request param — the only model in this catalog that documents it. + if (request.reasoningEffort !== undefined && request.reasoningEffort !== 'auto') { + payload.reasoning_effort = request.reasoningEffort + } + + // Z.ai's chat-completions API supports `text` and `json_object` response formats but + // does not have confirmed `json_schema` support, unlike the shared OpenAI-compatible + // template — request a plain JSON-object hint instead of a strict schema. + const responseFormatPayload = request.responseFormat + ? ({ type: 'json_object' as const } as const) + : undefined + + let preparedTools: ReturnType | null = null + let hasActiveTools = false + + if (tools?.length) { + preparedTools = prepareToolsWithUsageControl(tools, request.tools, logger, 'openai') + const { tools: filteredTools, toolChoice } = preparedTools + + if (filteredTools?.length && toolChoice) { + payload.tools = filteredTools + // Z.ai's chat-completions API documents `tool_choice` support for `"auto"` only — + // forcing a specific function or disabling tool use via the parameter is rejected. + // Ignore any forced/none choice `prepareToolsWithUsageControl` may have produced. + payload.tool_choice = 'auto' + hasActiveTools = true + + if (preparedTools.forcedTools.length > 0) { + logger.warn( + "Z.ai does not support forcing a specific tool via tool_choice (API only accepts 'auto') — ignoring force setting and falling back to auto", + { forcedTools: preparedTools.forcedTools, model: request.model } + ) + } + + logger.info('Z.ai request configuration:', { + toolCount: filteredTools.length, + toolChoice: 'auto', + model: request.model, + }) + } + } + + // Structured output and tool calling cannot be sent together — OpenAI-compatible + // backends reject a request that carries both `response_format` and active + // `tools`/`tool_choice`. Defer the schema until after the tool loop completes. + const deferResponseFormat = !!responseFormatPayload && hasActiveTools + if (responseFormatPayload && !deferResponseFormat) { + payload.response_format = responseFormatPayload + } + + if (request.stream && (!tools || tools.length === 0 || !hasActiveTools)) { + logger.info('Using streaming response for Z.ai request (no tools)') + + const streamResponse = await zai.chat.completions.create( + { + ...payload, + stream: true, + stream_options: { include_usage: true }, + }, + request.abortSignal ? { signal: request.abortSignal } : undefined + ) + + const streamingResult = createStreamingExecution({ + model: request.model, + providerStartTime, + providerStartTimeISO, + timing: { kind: 'simple', segmentName: request.model }, + initialTokens: { input: 0, output: 0, total: 0 }, + initialCost: { input: 0, output: 0, total: 0 }, + isStreaming: true, + createStream: ({ output }) => + createReadableStreamFromZaiStream(streamResponse as any, (content, usage) => { + output.content = content + output.tokens = { + input: usage.prompt_tokens, + output: usage.completion_tokens, + total: usage.total_tokens, + } + + const costResult = calculateCost( + request.model, + usage.prompt_tokens, + usage.completion_tokens + ) + output.cost = { + input: costResult.input, + output: costResult.output, + total: costResult.total, + } + }), + }) + + return streamingResult + } + + const initialCallTime = Date.now() + const originalToolChoice = payload.tool_choice + const forcedTools = preparedTools?.forcedTools || [] + let usedForcedTools: string[] = [] + + let currentResponse = await zai.chat.completions.create( + payload, + request.abortSignal ? { signal: request.abortSignal } : undefined + ) + const firstResponseTime = Date.now() - initialCallTime + + let content = currentResponse.choices[0]?.message?.content || '' + + const tokens = { + input: currentResponse.usage?.prompt_tokens || 0, + output: currentResponse.usage?.completion_tokens || 0, + total: currentResponse.usage?.total_tokens || 0, + } + const toolCalls = [] + const toolResults: Record[] = [] + const currentMessages = [...formattedMessages] + let iterationCount = 0 + let hasUsedForcedTool = false + let modelTime = firstResponseTime + let toolsTime = 0 + + const timeSegments: TimeSegment[] = [ + { + type: 'model', + name: request.model, + startTime: initialCallTime, + endTime: initialCallTime + firstResponseTime, + duration: firstResponseTime, + }, + ] + + if ( + typeof originalToolChoice === 'object' && + currentResponse.choices[0]?.message?.tool_calls + ) { + const toolCallsResponse = currentResponse.choices[0].message.tool_calls + const result = trackForcedToolUsage( + toolCallsResponse, + originalToolChoice, + logger, + 'openai', + forcedTools, + usedForcedTools + ) + hasUsedForcedTool = result.hasUsedForcedTool + usedForcedTools = result.usedForcedTools + } + + try { + while (iterationCount < MAX_TOOL_ITERATIONS) { + if (currentResponse.choices[0]?.message?.content) { + content = currentResponse.choices[0].message.content + } + + const toolCallsInResponse = currentResponse.choices[0]?.message?.tool_calls + + enrichLastModelSegmentFromChatCompletions( + timeSegments, + currentResponse, + toolCallsInResponse, + { model: request.model, provider: 'zai' } + ) + + if (!toolCallsInResponse || toolCallsInResponse.length === 0) { + break + } + + const toolsStartTime = Date.now() + + const toolExecutionPromises = toolCallsInResponse.map(async (toolCall) => { + const toolCallStartTime = Date.now() + const toolName = toolCall.function.name + + try { + const toolArgs = JSON.parse(toolCall.function.arguments) + const tool = request.tools?.find((t) => t.id === toolName) + + // Every tool_call in the assistant message must be answered by a matching + // `tool` message, or the next request violates the OpenAI message contract. + // Emit an error result for an unknown tool rather than dropping it. + if (!tool) { + const toolCallEndTime = Date.now() + return { + toolCall, + toolName, + toolParams: {}, + result: { + success: false, + output: undefined, + error: `Tool "${toolName}" is not available`, + }, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallEndTime - toolCallStartTime, + } + } + + const { toolParams, executionParams } = prepareToolExecution(tool, toolArgs, request) + const result = await executeTool(toolName, executionParams, { + signal: request.abortSignal, + }) + const toolCallEndTime = Date.now() + + return { + toolCall, + toolName, + toolParams, + result, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallEndTime - toolCallStartTime, + } + } catch (error) { + const toolCallEndTime = Date.now() + logger.error('Error processing tool call:', { error, toolName }) + + return { + toolCall, + toolName, + toolParams: {}, + result: { + success: false, + output: undefined, + error: getErrorMessage(error, 'Tool execution failed'), + }, + startTime: toolCallStartTime, + endTime: toolCallEndTime, + duration: toolCallEndTime - toolCallStartTime, + } + } + }) + + const executionResults = await Promise.allSettled(toolExecutionPromises) + + currentMessages.push({ + role: 'assistant', + content: null, + tool_calls: toolCallsInResponse.map((tc) => ({ + id: tc.id, + type: 'function', + function: { + name: tc.function.name, + arguments: tc.function.arguments, + }, + })), + }) + + for (const settledResult of executionResults) { + if (settledResult.status === 'rejected' || !settledResult.value) continue + + const { toolCall, toolName, toolParams, result, startTime, endTime, duration } = + settledResult.value + + timeSegments.push({ + type: 'tool', + name: toolName, + startTime: startTime, + endTime: endTime, + duration: duration, + toolCallId: toolCall.id, + }) + + let resultContent: any + if (result.success && result.output) { + toolResults.push(result.output) + resultContent = result.output + } else { + resultContent = { + error: true, + message: result.error || 'Tool execution failed', + tool: toolName, + } + } + + toolCalls.push({ + name: toolName, + arguments: toolParams, + startTime: new Date(startTime).toISOString(), + endTime: new Date(endTime).toISOString(), + duration: duration, + result: resultContent, + success: result.success, + }) + + currentMessages.push({ + role: 'tool', + tool_call_id: toolCall.id, + content: JSON.stringify(resultContent), + }) + } + + const thisToolsTime = Date.now() - toolsStartTime + toolsTime += thisToolsTime + + const nextPayload = { + ...payload, + messages: currentMessages, + } + + if ( + typeof originalToolChoice === 'object' && + hasUsedForcedTool && + forcedTools.length > 0 + ) { + const remainingTools = forcedTools.filter((tool) => !usedForcedTools.includes(tool)) + + if (remainingTools.length > 0) { + nextPayload.tool_choice = { + type: 'function', + function: { name: remainingTools[0] }, + } + logger.info(`Forcing next tool: ${remainingTools[0]}`) + } else { + nextPayload.tool_choice = 'auto' + logger.info('All forced tools have been used, switching to auto tool_choice') + } + } + + const nextModelStartTime = Date.now() + currentResponse = await zai.chat.completions.create( + nextPayload, + request.abortSignal ? { signal: request.abortSignal } : undefined + ) + + if ( + typeof nextPayload.tool_choice === 'object' && + currentResponse.choices[0]?.message?.tool_calls + ) { + const toolCallsResponse = currentResponse.choices[0].message.tool_calls + const result = trackForcedToolUsage( + toolCallsResponse, + nextPayload.tool_choice, + logger, + 'openai', + forcedTools, + usedForcedTools + ) + hasUsedForcedTool = result.hasUsedForcedTool + usedForcedTools = result.usedForcedTools + } + + const nextModelEndTime = Date.now() + const thisModelTime = nextModelEndTime - nextModelStartTime + + timeSegments.push({ + type: 'model', + name: request.model, + startTime: nextModelStartTime, + endTime: nextModelEndTime, + duration: thisModelTime, + }) + + modelTime += thisModelTime + + if (currentResponse.choices[0]?.message?.content) { + content = currentResponse.choices[0].message.content + } + + if (currentResponse.usage) { + tokens.input += currentResponse.usage.prompt_tokens || 0 + tokens.output += currentResponse.usage.completion_tokens || 0 + tokens.total += currentResponse.usage.total_tokens || 0 + } + + iterationCount++ + } + + if (iterationCount === MAX_TOOL_ITERATIONS) { + enrichLastModelSegmentFromChatCompletions( + timeSegments, + currentResponse, + currentResponse.choices[0]?.message?.tool_calls, + { model: request.model, provider: 'zai' } + ) + } + } catch (error) { + logger.error('Error in Z.ai request:', { error }) + throw error + } + + if (request.stream) { + logger.info('Using streaming for final Z.ai response after tool processing') + + // The tool loop is complete: this final pass only produces the textual answer. + // Z.ai only documents `tool_choice: 'auto'` support, so drop `tools`/`tool_choice` + // entirely rather than sending an unsupported `'none'` — with no tools declared, + // the model has nothing to call and the text-only stream adapter stays safe. + const streamingPayload: any = { + ...payload, + messages: currentMessages, + stream: true, + stream_options: { include_usage: true }, + } + streamingPayload.tools = undefined + streamingPayload.tool_choice = undefined + if (deferResponseFormat && responseFormatPayload) { + streamingPayload.response_format = responseFormatPayload + } + + const streamResponse = await zai.chat.completions.create( + streamingPayload, + request.abortSignal ? { signal: request.abortSignal } : undefined + ) + + const accumulatedCost = calculateCost(request.model, tokens.input, tokens.output) + + const streamingResult = createStreamingExecution({ + model: request.model, + providerStartTime, + providerStartTimeISO, + timing: { + kind: 'accumulated', + modelTime, + toolsTime, + firstResponseTime, + iterations: iterationCount + 1, + timeSegments, + }, + initialTokens: { + input: tokens.input, + output: tokens.output, + total: tokens.total, + }, + initialCost: { + input: accumulatedCost.input, + output: accumulatedCost.output, + toolCost: undefined as number | undefined, + total: accumulatedCost.total, + }, + toolCalls: + toolCalls.length > 0 + ? { + list: toolCalls, + count: toolCalls.length, + } + : undefined, + isStreaming: true, + createStream: ({ output }) => + createReadableStreamFromZaiStream(streamResponse as any, (content, usage) => { + output.content = content + output.tokens = { + input: tokens.input + usage.prompt_tokens, + output: tokens.output + usage.completion_tokens, + total: tokens.total + usage.total_tokens, + } + + const streamCost = calculateCost( + request.model, + usage.prompt_tokens, + usage.completion_tokens + ) + const tc = sumToolCosts(toolResults) + output.cost = { + input: accumulatedCost.input + streamCost.input, + output: accumulatedCost.output + streamCost.output, + toolCost: tc || undefined, + total: accumulatedCost.total + streamCost.total + tc, + } + }), + }) + + return streamingResult + } + + // Tools were active, so `response_format` was withheld from the loop. Make one final + // tool-free call to obtain the structured response now that the tool work is done. + if (deferResponseFormat && responseFormatPayload) { + logger.info('Applying deferred response_format after tool processing') + + const finalFormatStartTime = Date.now() + // Z.ai only documents `tool_choice: 'auto'` support, so drop `tools`/`tool_choice` + // rather than sending an unsupported `'none'` — with no tools declared, the model + // has nothing left to call and simply returns the formatted answer. + const finalPayload: any = { + ...payload, + messages: currentMessages, + response_format: responseFormatPayload, + } + finalPayload.tools = undefined + finalPayload.tool_choice = undefined + + currentResponse = await zai.chat.completions.create( + finalPayload, + request.abortSignal ? { signal: request.abortSignal } : undefined + ) + + const finalFormatEndTime = Date.now() + timeSegments.push({ + type: 'model', + name: request.model, + startTime: finalFormatStartTime, + endTime: finalFormatEndTime, + duration: finalFormatEndTime - finalFormatStartTime, + }) + modelTime += finalFormatEndTime - finalFormatStartTime + + const formattedContent = currentResponse.choices[0]?.message?.content + if (formattedContent) { + content = formattedContent + } + + if (currentResponse.usage) { + tokens.input += currentResponse.usage.prompt_tokens || 0 + tokens.output += currentResponse.usage.completion_tokens || 0 + tokens.total += currentResponse.usage.total_tokens || 0 + } + + enrichLastModelSegmentFromChatCompletions( + timeSegments, + currentResponse, + currentResponse.choices[0]?.message?.tool_calls, + { model: request.model, provider: 'zai' } + ) + } + + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime + + return { + content, + model: request.model, + tokens, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + toolResults: toolResults.length > 0 ? toolResults : undefined, + timing: { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + modelTime: modelTime, + toolsTime: toolsTime, + firstResponseTime: firstResponseTime, + iterations: iterationCount + 1, + timeSegments: timeSegments, + }, + } + } catch (error) { + const providerEndTime = Date.now() + const providerEndTimeISO = new Date(providerEndTime).toISOString() + const totalDuration = providerEndTime - providerStartTime + + logger.error('Error in Z.ai request:', { + error, + duration: totalDuration, + }) + + throw new ProviderError(toError(error).message, { + startTime: providerStartTimeISO, + endTime: providerEndTimeISO, + duration: totalDuration, + }) + } + }, +} diff --git a/apps/sim/providers/zai/utils.ts b/apps/sim/providers/zai/utils.ts new file mode 100644 index 00000000000..4b86c6b7619 --- /dev/null +++ b/apps/sim/providers/zai/utils.ts @@ -0,0 +1,14 @@ +import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import type { CompletionUsage } from 'openai/resources/completions' +import { createOpenAICompatibleStream } from '@/providers/utils' + +/** + * Creates a ReadableStream from a Z.ai streaming response. + * Uses the shared OpenAI-compatible streaming utility. + */ +export function createReadableStreamFromZaiStream( + zaiStream: AsyncIterable, + onComplete?: (content: string, usage: CompletionUsage) => void +): ReadableStream { + return createOpenAICompatibleStream(zaiStream, 'Z.ai', onComplete) +} diff --git a/apps/sim/tools/types.ts b/apps/sim/tools/types.ts index dcce33cbe20..910e825f251 100644 --- a/apps/sim/tools/types.ts +++ b/apps/sim/tools/types.ts @@ -7,6 +7,7 @@ export type BYOKProviderId = | 'anthropic' | 'google' | 'mistral' + | 'zai' | 'fireworks' | 'together' | 'baseten'