From 07ce99dc9e0fda20a16f871d9470e7fcb6de28b9 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Sun, 19 Jul 2026 16:08:26 +0000 Subject: [PATCH 1/3] feat(ai): add Alibaba image generation --- packages/ai/src/protocols/alibaba-images.ts | 338 ++++++++++++++++++ packages/ai/src/protocols/index.ts | 1 + packages/ai/src/providers/alibaba.ts | 48 +++ packages/ai/src/providers/index.ts | 1 + .../generates-with-qwen-image-2-0.json | 28 ++ .../generates-with-wan-2-7.json | 28 ++ .../provider/alibaba-images.recorded.test.ts | 47 +++ .../ai/test/provider/alibaba-images.test.ts | 196 ++++++++++ 8 files changed, 687 insertions(+) create mode 100644 packages/ai/src/protocols/alibaba-images.ts create mode 100644 packages/ai/src/providers/alibaba.ts create mode 100644 packages/ai/test/fixtures/recordings/alibaba-images/generates-with-qwen-image-2-0.json create mode 100644 packages/ai/test/fixtures/recordings/alibaba-images/generates-with-wan-2-7.json create mode 100644 packages/ai/test/provider/alibaba-images.recorded.test.ts create mode 100644 packages/ai/test/provider/alibaba-images.test.ts diff --git a/packages/ai/src/protocols/alibaba-images.ts b/packages/ai/src/protocols/alibaba-images.ts new file mode 100644 index 000000000000..45282a014bdf --- /dev/null +++ b/packages/ai/src/protocols/alibaba-images.ts @@ -0,0 +1,338 @@ +import { Effect, Schema } from "effect" +import { Headers, HttpClientRequest } from "effect/unstable/http" +import { + GeneratedImage, + ImageModel, + ImageResponse, + type ImageModelDefaults, + type ImageRequest, + type ImageRoute, +} from "../image" +import { Auth, type Definition as AuthDefinition } from "../route/auth" +import { + InvalidProviderOutputReason, + LLMError, + UnknownProviderReason, + Usage, + mergeHttpOptions, + mergeJsonRecords, +} from "../schema" +import { ProviderShared } from "./shared" + +const ADAPTER = "alibaba-images" +export const DEFAULT_BASE_URL = "https://dashscope-intl.aliyuncs.com/api/v1" +export const PATH = "/services/aigc/multimodal-generation/generation" + +export type Family = "qwen" | "wan" + +export interface QwenImageOptions extends Record { + readonly negativePrompt?: string + readonly promptExtend?: boolean + readonly watermark?: boolean +} + +export interface WanColor { + readonly hex: string + readonly ratio: string +} + +export interface WanImageOptions extends Record { + readonly resolution?: "1K" | "2K" | "4K" + readonly thinkingMode?: boolean + readonly colorPalette?: ReadonlyArray + readonly watermark?: boolean +} + +export interface AlibabaImageOptions extends Record { + readonly qwen?: QwenImageOptions + readonly wan?: WanImageOptions +} + +declare module "../image" { + interface ImageProviderOptions { + readonly alibaba?: AlibabaImageOptions + } +} + +const MessageInput = Schema.Struct({ + messages: Schema.Tuple([ + Schema.Struct({ + role: Schema.tag("user"), + content: Schema.Tuple([Schema.Struct({ text: Schema.String })]), + }), + ]), +}) + +const QwenBody = Schema.Struct({ + model: Schema.String, + input: MessageInput, + parameters: Schema.Struct({ + size: Schema.optional(Schema.String), + n: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(1))), + negative_prompt: Schema.optional(Schema.String), + prompt_extend: Schema.optional(Schema.Boolean), + watermark: Schema.optional(Schema.Boolean), + seed: Schema.optional(Schema.Int), + }), +}) +export type QwenBody = Schema.Schema.Type + +const WanBody = Schema.Struct({ + model: Schema.String, + input: MessageInput, + parameters: Schema.Struct({ + size: Schema.optional(Schema.String), + n: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(1))), + thinking_mode: Schema.optional(Schema.Boolean), + color_palette: Schema.optional( + Schema.Array( + Schema.Struct({ + hex: Schema.String, + ratio: Schema.String, + }), + ), + ), + watermark: Schema.optional(Schema.Boolean), + seed: Schema.optional(Schema.Int), + }), +}) +export type WanBody = Schema.Schema.Type + +const AlibabaResponse = Schema.Struct({ + output: Schema.optional( + Schema.Struct({ + choices: Schema.Array( + Schema.Struct({ + finish_reason: Schema.optional(Schema.String), + message: Schema.Struct({ + role: Schema.optional(Schema.String), + content: Schema.Array( + Schema.Struct({ + image: Schema.String, + type: Schema.optional(Schema.String), + }), + ), + }), + }), + ), + finished: Schema.optional(Schema.Boolean), + }), + ), + usage: Schema.optional( + Schema.Struct({ + image_count: Schema.optional(Schema.Number), + input_tokens: Schema.optional(Schema.Number), + output_tokens: Schema.optional(Schema.Number), + total_tokens: Schema.optional(Schema.Number), + width: Schema.optional(Schema.Number), + height: Schema.optional(Schema.Number), + size: Schema.optional(Schema.String), + }), + ), + request_id: Schema.optional(Schema.String), + code: Schema.optional(Schema.String), + message: Schema.optional(Schema.String), +}) + +export interface ModelInput { + readonly id: string + readonly family: Family + readonly auth: AuthDefinition + readonly baseURL?: string + readonly headers?: Record + readonly defaults?: ImageModelDefaults +} + +const options = (request: ImageRequest): AlibabaImageOptions => ({ + ...request.model.defaults?.providerOptions?.alibaba, + ...request.providerOptions?.alibaba, + qwen: { + ...(request.model.defaults?.providerOptions?.alibaba?.qwen as QwenImageOptions | undefined), + ...(request.providerOptions?.alibaba?.qwen as QwenImageOptions | undefined), + }, + wan: { + ...(request.model.defaults?.providerOptions?.alibaba?.wan as WanImageOptions | undefined), + ...(request.providerOptions?.alibaba?.wan as WanImageOptions | undefined), + }, +}) + +const messageInput = (request: ImageRequest) => ({ + messages: [{ role: "user" as const, content: [{ text: request.prompt }] }] as const, +}) + +const qwenBody = (request: ImageRequest): QwenBody => { + const qwen = options(request).qwen + return { + model: request.model.id, + input: messageInput(request), + parameters: { + size: request.size === undefined ? undefined : `${request.size.width}*${request.size.height}`, + n: request.count, + negative_prompt: qwen?.negativePrompt, + prompt_extend: qwen?.promptExtend, + watermark: qwen?.watermark, + seed: request.seed, + }, + } +} + +const wanBody = (request: ImageRequest): WanBody => { + const wan = options(request).wan + return { + model: request.model.id, + input: messageInput(request), + parameters: { + size: + wan?.resolution ?? (request.size === undefined ? undefined : `${request.size.width}*${request.size.height}`), + n: request.count, + thinking_mode: wan?.thinkingMode, + color_palette: wan?.colorPalette === undefined ? undefined : [...wan.colorPalette], + watermark: wan?.watermark, + seed: request.seed, + }, + } +} + +const invalidOutput = (message: string, metadata?: Record) => + new LLMError({ + module: ADAPTER, + method: "generate", + reason: new InvalidProviderOutputReason({ + message, + route: ADAPTER, + providerMetadata: metadata === undefined ? undefined : { alibaba: metadata }, + }), + }) + +const providerError = (code: string | undefined, message: string | undefined, requestID: string | undefined) => + new LLMError({ + module: ADAPTER, + method: "generate", + reason: new UnknownProviderReason({ + message: [code, message].filter(Boolean).join(": ") || "Alibaba image generation failed", + providerMetadata: { alibaba: { code, requestId: requestID } }, + }), + }) + +const applyQuery = (url: string, query: Record | undefined) => { + if (!query) return url + const next = new URL(url) + Object.entries(query).forEach(([key, value]) => next.searchParams.set(key, value)) + return next.toString() +} + +const PROTOCOL_BODY_FIELDS = new Set(["model", "input", "parameters"]) + +const bodyWithOverlay = Effect.fn("AlibabaImages.bodyWithOverlay")(function* ( + imageBody: QwenBody | WanBody, + overlay: Record | undefined, +) { + if (!overlay) return imageBody + const reserved = Object.keys(overlay).filter((key) => PROTOCOL_BODY_FIELDS.has(key)) + if (reserved.length > 0) + return yield* ProviderShared.invalidRequest( + `http.body cannot overlay protocol-owned field(s): ${reserved.join(", ")}`, + ) + return mergeJsonRecords(imageBody, overlay) ?? imageBody +}) + +const expiration = (url: string) => { + if (!URL.canParse(url)) return undefined + const value = new URL(url).searchParams.get("Expires") + if (value === null || !Number.isFinite(Number(value))) return undefined + return new Date(Number(value) * 1000).toISOString() +} + +export const model = (input: ModelInput) => { + const route: ImageRoute = { + id: `${ADAPTER}-${input.family}`, + generate: Effect.fn("AlibabaImages.generate")(function* (request: ImageRequest, execute) { + if (request.aspectRatio !== undefined) + return yield* ProviderShared.invalidRequest("Alibaba Images does not support the common aspectRatio option") + if ( + input.family === "qwen" && + (request.model.defaults?.providerOptions?.alibaba?.wan !== undefined || + request.providerOptions?.alibaba?.wan !== undefined) + ) + return yield* ProviderShared.invalidRequest("Qwen Image does not accept providerOptions.alibaba.wan") + if ( + input.family === "wan" && + (request.model.defaults?.providerOptions?.alibaba?.qwen !== undefined || + request.providerOptions?.alibaba?.qwen !== undefined) + ) + return yield* ProviderShared.invalidRequest("Wan Image does not accept providerOptions.alibaba.qwen") + if (input.family === "wan" && request.size !== undefined && options(request).wan?.resolution !== undefined) + return yield* ProviderShared.invalidRequest("Wan Image accepts either size or resolution, not both") + + const requestBody = yield* ProviderShared.validateWith( + Schema.decodeUnknownEffect(input.family === "qwen" ? QwenBody : WanBody), + )(input.family === "qwen" ? qwenBody(request) : wanBody(request)) + const http = mergeHttpOptions(request.model.defaults?.http, request.http) + const overlaidBody = yield* bodyWithOverlay(requestBody, http?.body) + const text = ProviderShared.encodeJson(overlaidBody) + const url = applyQuery(`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}${PATH}`, http?.query) + const headers = yield* Auth.toEffect(input.auth)({ + request, + method: "POST", + url, + body: text, + headers: Headers.fromInput({ ...input.headers, ...http?.headers }), + }) + const response = yield* execute( + HttpClientRequest.post(url).pipe( + HttpClientRequest.setHeaders(headers), + HttpClientRequest.bodyText(text, "application/json"), + ), + ) + const payload = yield* response.json.pipe( + Effect.mapError(() => invalidOutput("Failed to read the Alibaba Images response")), + ) + const decoded = yield* Schema.decodeUnknownEffect(AlibabaResponse)(payload).pipe( + Effect.mapError(() => invalidOutput("Alibaba Images returned an invalid response")), + ) + if (decoded.code !== undefined || decoded.output === undefined) + return yield* providerError(decoded.code, decoded.message, decoded.request_id) + const urls = decoded.output.choices.flatMap((choice) => choice.message.content.map((content) => content.image)) + if (urls.length === 0) + return yield* invalidOutput("Alibaba Images returned no images", { requestId: decoded.request_id }) + + return new ImageResponse({ + images: urls.map( + (url) => + new GeneratedImage({ + mediaType: "image/png", + data: url, + providerMetadata: { + alibaba: { + modelId: request.model.id, + family: input.family, + expiresAt: expiration(url), + }, + }, + }), + ), + usage: + decoded.usage === undefined + ? undefined + : new Usage({ + inputTokens: decoded.usage.input_tokens, + outputTokens: decoded.usage.output_tokens, + totalTokens: decoded.usage.total_tokens, + providerMetadata: { alibaba: decoded.usage }, + }), + providerMetadata: { + alibaba: { + requestId: decoded.request_id, + modelId: request.model.id, + family: input.family, + }, + }, + }) + }), + } + return ImageModel.make({ id: input.id, provider: "alibaba", route, defaults: input.defaults }) +} + +export const AlibabaImages = { + model, +} as const diff --git a/packages/ai/src/protocols/index.ts b/packages/ai/src/protocols/index.ts index bd3ee8eeec1d..37f1c6100584 100644 --- a/packages/ai/src/protocols/index.ts +++ b/packages/ai/src/protocols/index.ts @@ -1,4 +1,5 @@ export * as AnthropicMessages from "./anthropic-messages" +export * as AlibabaImages from "./alibaba-images" export * as BedrockConverse from "./bedrock-converse" export * as Gemini from "./gemini" export * as OpenAIChat from "./openai-chat" diff --git a/packages/ai/src/providers/alibaba.ts b/packages/ai/src/providers/alibaba.ts new file mode 100644 index 000000000000..b4cfddfb87f2 --- /dev/null +++ b/packages/ai/src/providers/alibaba.ts @@ -0,0 +1,48 @@ +import type { ImageModel } from "../image" +import { AlibabaImages, type AlibabaImageOptions, type Family } from "../protocols/alibaba-images" +import { AuthOptions, type ProviderAuthOption } from "../route/auth-options" +import { HttpOptions, ProviderID, mergeHttpOptions } from "../schema" + +export type { AlibabaImageOptions, QwenImageOptions, WanColor, WanImageOptions } from "../protocols/alibaba-images" + +export type AlibabaImageModelID = "qwen-image-2.0" | "qwen-image-2.0-pro" | "wan2.7-image" | "wan2.7-image-pro" + +export const id = ProviderID.make("alibaba") + +export type Config = ProviderAuthOption<"optional"> & { + readonly baseURL?: string + readonly headers?: Record + readonly http?: HttpOptions.Input + readonly image?: { + readonly providerOptions?: AlibabaImageOptions + } +} + +const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "DASHSCOPE_API_KEY") + +const family = (modelID: AlibabaImageModelID): Family => (modelID.startsWith("qwen-") ? "qwen" : "wan") + +export const configure = (input: Config = {}) => { + const image = (modelID: AlibabaImageModelID): ImageModel => + AlibabaImages.model({ + id: modelID, + family: family(modelID), + auth: auth(input), + baseURL: input.baseURL, + headers: input.headers, + defaults: { + providerOptions: + input.image?.providerOptions === undefined ? undefined : { alibaba: { ...input.image.providerOptions } }, + http: mergeHttpOptions(input.http === undefined ? undefined : HttpOptions.make(input.http)), + }, + }) + + return { + id, + image, + configure, + } +} + +export const provider = configure() +export const image = provider.image diff --git a/packages/ai/src/providers/index.ts b/packages/ai/src/providers/index.ts index 8b794f9a3026..4c2139a6e09b 100644 --- a/packages/ai/src/providers/index.ts +++ b/packages/ai/src/providers/index.ts @@ -1,6 +1,7 @@ export * as Anthropic from "./anthropic" export * as AnthropicCompatible from "./anthropic-compatible" export * as AmazonBedrock from "./amazon-bedrock" +export * as Alibaba from "./alibaba" export * as Azure from "./azure" export * as Cloudflare from "./cloudflare" export { CloudflareAIGateway, CloudflareWorkersAI } from "./cloudflare" diff --git a/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-qwen-image-2-0.json b/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-qwen-image-2-0.json new file mode 100644 index 000000000000..8c051c7532fd --- /dev/null +++ b/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-qwen-image-2-0.json @@ -0,0 +1,28 @@ +{ + "version": 1, + "metadata": { + "tags": ["prefix:alibaba-images", "provider:alibaba", "protocol:alibaba-images"], + "name": "alibaba-images/generates-with-qwen-image-2-0", + "recordedAt": "2026-07-19T16:05:19.642Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"qwen-image-2.0\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"A simple flat black circle centered on a plain white background.\"}]}]},\"parameters\":{\"size\":\"512*512\",\"prompt_extend\":false,\"watermark\":false}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/json" + }, + "body": "{\"output\":{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":[{\"image\":\"https://dashscope-463f.oss-accelerate.aliyuncs.com/7d/7b/20260720/991d10e3/fe425a64-dc6f-4c67-a4fa-a06810984ba9.png?Expires=1785082919&OSSAccessKeyId=fixture&Signature=fixture\"}],\"role\":\"assistant\"}}]},\"usage\":{\"height\":512,\"image_count\":1,\"width\":512},\"request_id\":\"484490f4-9881-90eb-8d52-d270d2395920\"}" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-wan-2-7.json b/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-wan-2-7.json new file mode 100644 index 000000000000..373482ec134e --- /dev/null +++ b/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-wan-2-7.json @@ -0,0 +1,28 @@ +{ + "version": 1, + "metadata": { + "tags": ["prefix:alibaba-images", "provider:alibaba", "protocol:alibaba-images"], + "name": "alibaba-images/generates-with-wan-2-7", + "recordedAt": "2026-07-19T16:05:23.617Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"wan2.7-image\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":[{\"text\":\"A simple flat black square centered on a plain white background.\"}]}]},\"parameters\":{\"size\":\"1K\",\"thinking_mode\":false,\"watermark\":false}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/json" + }, + "body": "{\"output\":{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":[{\"image\":\"https://dashscope-463f.oss-accelerate.aliyuncs.com/1d/ae/20260720/10739b93/1784477122_595381b5.png?Expires=1784563522&OSSAccessKeyId=fixture&Signature=fixture\",\"type\":\"image\"}],\"role\":\"assistant\"}}],\"finished\":true},\"usage\":{\"image_count\":1,\"input_tokens\":31,\"output_tokens\":2,\"size\":\"1024*1024\",\"total_tokens\":33},\"request_id\":\"ec5cacb5-c7b0-9e46-8459-49003a3816c3\"}" + } + } + ] +} diff --git a/packages/ai/test/provider/alibaba-images.recorded.test.ts b/packages/ai/test/provider/alibaba-images.recorded.test.ts new file mode 100644 index 000000000000..4980cd81b169 --- /dev/null +++ b/packages/ai/test/provider/alibaba-images.recorded.test.ts @@ -0,0 +1,47 @@ +import { describe, expect } from "bun:test" +import { Effect } from "effect" +import { Image } from "../../src" +import { Alibaba } from "../../src/providers" +import { recordedTests } from "../recorded-test" + +const alibaba = Alibaba.configure({ + apiKey: process.env.DASHSCOPE_API_KEY ?? "fixture", +}) + +const recorded = recordedTests({ + prefix: "alibaba-images", + provider: "alibaba", + protocol: "alibaba-images", + requires: ["DASHSCOPE_API_KEY"], +}) + +describe("Alibaba Images recorded", () => { + recorded.effect("generates with Qwen Image 2.0", () => + Effect.gen(function* () { + const response = yield* Image.generate({ + model: alibaba.image("qwen-image-2.0"), + prompt: "A simple flat black circle centered on a plain white background.", + size: { width: 512, height: 512 }, + providerOptions: { alibaba: { qwen: { promptExtend: false, watermark: false } } }, + }) + + expect(response.images).toHaveLength(1) + expect(response.image?.mediaType).toBe("image/png") + expect(response.image?.data).toStartWith("https://") + }), + ) + + recorded.effect("generates with Wan 2.7", () => + Effect.gen(function* () { + const response = yield* Image.generate({ + model: alibaba.image("wan2.7-image"), + prompt: "A simple flat black square centered on a plain white background.", + providerOptions: { alibaba: { wan: { resolution: "1K", thinkingMode: false, watermark: false } } }, + }) + + expect(response.images).toHaveLength(1) + expect(response.image?.mediaType).toBe("image/png") + expect(response.image?.data).toStartWith("https://") + }), + ) +}) diff --git a/packages/ai/test/provider/alibaba-images.test.ts b/packages/ai/test/provider/alibaba-images.test.ts new file mode 100644 index 000000000000..d8bfd12b93d3 --- /dev/null +++ b/packages/ai/test/provider/alibaba-images.test.ts @@ -0,0 +1,196 @@ +import { describe, expect } from "bun:test" +import { Effect, Layer } from "effect" +import { HttpClientRequest } from "effect/unstable/http" +import { Image, ImageClient } from "../../src" +import { Alibaba } from "../../src/providers" +import { it } from "../lib/effect" +import { dynamicResponse } from "../lib/http" + +const response = (family: "qwen" | "wan") => ({ + output: { + choices: [ + { + finish_reason: "stop", + message: { + role: "assistant", + content: [ + { + image: "https://dashscope-result-intl.oss-cn-singapore.aliyuncs.com/result.png?Expires=1893456000", + ...(family === "wan" ? { type: "image" } : {}), + }, + ], + }, + }, + ], + ...(family === "wan" ? { finished: true } : {}), + }, + usage: + family === "wan" + ? { image_count: 1, input_tokens: 10, output_tokens: 2, total_tokens: 12, size: "2048*2048" } + : { image_count: 1, width: 1024, height: 768 }, + request_id: `request-${family}`, +}) + +describe("Alibaba Images", () => { + it.effect("generates Qwen Image 2.0 through the international synchronous route", () => + Effect.gen(function* () { + const result = yield* Image.generate({ + model: Alibaba.configure({ + apiKey: "test", + baseURL: "https://dashscope-intl.test/api/v1", + image: { providerOptions: { qwen: { promptExtend: true, watermark: false } } }, + http: { headers: { "x-default": "yes" } }, + }).image("qwen-image-2.0-pro"), + prompt: "A robot tending a rooftop garden", + count: 2, + size: { width: 1024, height: 768 }, + seed: 42, + providerOptions: { alibaba: { qwen: { negativePrompt: "blurry" } } }, + http: { headers: { "x-request": "yes" }, query: { trace: "1" }, body: { metadata: "test" } }, + }) + + expect(result.image?.data).toContain("dashscope-result-intl") + expect(result.image?.providerMetadata).toEqual({ + alibaba: { + modelId: "qwen-image-2.0-pro", + family: "qwen", + expiresAt: "2030-01-01T00:00:00.000Z", + }, + }) + expect(result.providerMetadata).toEqual({ + alibaba: { requestId: "request-qwen", modelId: "qwen-image-2.0-pro", family: "qwen" }, + }) + }).pipe( + Effect.provide( + ImageClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => + Effect.gen(function* () { + const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie) + expect(request.url).toBe( + "https://dashscope-intl.test/api/v1/services/aigc/multimodal-generation/generation?trace=1", + ) + expect(request.headers.get("authorization")).toBe("Bearer test") + expect(request.headers.get("x-default")).toBe("yes") + expect(request.headers.get("x-request")).toBe("yes") + expect(JSON.parse(input.text)).toEqual({ + model: "qwen-image-2.0-pro", + input: { + messages: [{ role: "user", content: [{ text: "A robot tending a rooftop garden" }] }], + }, + parameters: { + size: "1024*768", + n: 2, + negative_prompt: "blurry", + prompt_extend: true, + watermark: false, + seed: 42, + }, + metadata: "test", + }) + return input.respond(JSON.stringify(response("qwen")), { + headers: { "content-type": "application/json" }, + }) + }), + ), + ), + ), + ), + ), + ) + + it.effect("generates Wan 2.7 through the international synchronous route", () => + Effect.gen(function* () { + const result = yield* Image.generate({ + model: Alibaba.configure({ apiKey: "test", baseURL: "https://dashscope-intl.test/api/v1" }).image( + "wan2.7-image-pro", + ), + prompt: "A flower shop with a wooden door", + count: 1, + seed: 7, + providerOptions: { + alibaba: { + wan: { + resolution: "2K", + thinkingMode: true, + watermark: false, + colorPalette: [ + { hex: "#112233", ratio: "60.00%" }, + { hex: "#445566", ratio: "25.00%" }, + { hex: "#778899", ratio: "15.00%" }, + ], + }, + }, + }, + }) + + expect(result.images).toHaveLength(1) + expect(result.image?.mediaType).toBe("image/png") + expect(result.usage?.totalTokens).toBe(12) + expect(result.usage?.providerMetadata).toEqual({ alibaba: response("wan").usage }) + }).pipe( + Effect.provide( + ImageClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => + Effect.gen(function* () { + const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie) + expect(request.url).toBe( + "https://dashscope-intl.test/api/v1/services/aigc/multimodal-generation/generation", + ) + expect(JSON.parse(input.text)).toEqual({ + model: "wan2.7-image-pro", + input: { messages: [{ role: "user", content: [{ text: "A flower shop with a wooden door" }] }] }, + parameters: { + size: "2K", + n: 1, + thinking_mode: true, + color_palette: [ + { hex: "#112233", ratio: "60.00%" }, + { hex: "#445566", ratio: "25.00%" }, + { hex: "#778899", ratio: "15.00%" }, + ], + watermark: false, + seed: 7, + }, + }) + return input.respond(JSON.stringify(response("wan")), { + headers: { "content-type": "application/json" }, + }) + }), + ), + ), + ), + ), + ), + ) + + it.effect("surfaces Alibaba error envelopes as typed provider errors", () => + Image.generate({ + model: Alibaba.configure({ apiKey: "test" }).image("qwen-image-2.0"), + prompt: "A robot", + }).pipe( + Effect.flip, + Effect.tap((error) => + Effect.sync(() => { + expect(error.reason._tag).toBe("UnknownProvider") + expect(error.message).toContain("InvalidParameter: invalid prompt") + }), + ), + Effect.provide( + ImageClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => + Effect.succeed( + input.respond( + JSON.stringify({ request_id: "request-error", code: "InvalidParameter", message: "invalid prompt" }), + { headers: { "content-type": "application/json" } }, + ), + ), + ), + ), + ), + ), + ), + ) +}) From 0baabcdc4bc211b5900f1709108ed428c4ac04d2 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Sun, 19 Jul 2026 16:17:09 +0000 Subject: [PATCH 2/3] fix(ai): address Alibaba image audit findings --- packages/ai/src/protocols/alibaba-images.ts | 36 +++++----- packages/ai/test/auth-options.types.ts | 7 ++ .../generates-with-qwen-image-2-0.json | 4 +- .../generates-with-wan-2-7.json | 4 +- .../provider/alibaba-images.recorded.test.ts | 70 ++++++++++++++++++- .../ai/test/provider/alibaba-images.test.ts | 51 ++++++++++++++ 6 files changed, 149 insertions(+), 23 deletions(-) diff --git a/packages/ai/src/protocols/alibaba-images.ts b/packages/ai/src/protocols/alibaba-images.ts index 45282a014bdf..23ae5d8b618f 100644 --- a/packages/ai/src/protocols/alibaba-images.ts +++ b/packages/ai/src/protocols/alibaba-images.ts @@ -25,7 +25,7 @@ export const PATH = "/services/aigc/multimodal-generation/generation" export type Family = "qwen" | "wan" -export interface QwenImageOptions extends Record { +export interface QwenImageOptions { readonly negativePrompt?: string readonly promptExtend?: boolean readonly watermark?: boolean @@ -36,14 +36,14 @@ export interface WanColor { readonly ratio: string } -export interface WanImageOptions extends Record { +export interface WanImageOptions { readonly resolution?: "1K" | "2K" | "4K" readonly thinkingMode?: boolean readonly colorPalette?: ReadonlyArray readonly watermark?: boolean } -export interface AlibabaImageOptions extends Record { +export interface AlibabaImageOptions { readonly qwen?: QwenImageOptions readonly wan?: WanImageOptions } @@ -240,7 +240,8 @@ const expiration = (url: string) => { if (!URL.canParse(url)) return undefined const value = new URL(url).searchParams.get("Expires") if (value === null || !Number.isFinite(Number(value))) return undefined - return new Date(Number(value) * 1000).toISOString() + const date = new Date(Number(value) * 1000) + return Number.isNaN(date.getTime()) ? undefined : date.toISOString() } export const model = (input: ModelInput) => { @@ -297,20 +298,21 @@ export const model = (input: ModelInput) => { return yield* invalidOutput("Alibaba Images returned no images", { requestId: decoded.request_id }) return new ImageResponse({ - images: urls.map( - (url) => - new GeneratedImage({ - mediaType: "image/png", - data: url, - providerMetadata: { - alibaba: { - modelId: request.model.id, - family: input.family, - expiresAt: expiration(url), - }, + images: urls.map((url) => { + const expiresAt = expiration(url) + return new GeneratedImage({ + mediaType: "image/png", + data: url, + expiresAt, + providerMetadata: { + alibaba: { + modelId: request.model.id, + family: input.family, + expiresAt, }, - }), - ), + }, + }) + }), usage: decoded.usage === undefined ? undefined diff --git a/packages/ai/test/auth-options.types.ts b/packages/ai/test/auth-options.types.ts index c43010678995..2a7e9626c982 100644 --- a/packages/ai/test/auth-options.types.ts +++ b/packages/ai/test/auth-options.types.ts @@ -2,6 +2,7 @@ import { Config } from "effect" import { Auth } from "../src/route" import type { ModelFactory } from "../src/route/auth-options" import * as OpenAIChat from "../src/protocols/openai-chat" +import * as Alibaba from "../src/providers/alibaba" import * as AmazonBedrock from "../src/providers/amazon-bedrock" import * as Anthropic from "../src/providers/anthropic" import * as AnthropicCompatible from "../src/providers/anthropic-compatible" @@ -250,3 +251,9 @@ Cloudflare.CloudflareWorkersAI.configure({ accountId: "account", apiKey: "cf-key GitHubCopilot.configure({ baseURL: "https://copilot.test", apiKey: "copilot-key" }).model("gpt-4.1") // @ts-expect-error GitHub Copilot model selectors only accept model ids. GitHubCopilot.configure({ baseURL: "https://copilot.test", apiKey: "copilot-key" }).model("gpt-4.1", {}) + +Alibaba.configure({ image: { providerOptions: { qwen: { promptExtend: true } } } }).image("qwen-image-2.0") +// @ts-expect-error Alibaba Qwen options are closed and reject unimplemented parameters. +Alibaba.configure({ image: { providerOptions: { qwen: { style: "photorealistic" } } } }) +// @ts-expect-error Alibaba Wan options are closed and reject unimplemented parameters. +Alibaba.configure({ image: { providerOptions: { wan: { parameters: { watermark: true } } } } }) diff --git a/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-qwen-image-2-0.json b/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-qwen-image-2-0.json index 8c051c7532fd..648688aa1790 100644 --- a/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-qwen-image-2-0.json +++ b/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-qwen-image-2-0.json @@ -3,7 +3,7 @@ "metadata": { "tags": ["prefix:alibaba-images", "provider:alibaba", "protocol:alibaba-images"], "name": "alibaba-images/generates-with-qwen-image-2-0", - "recordedAt": "2026-07-19T16:05:19.642Z" + "recordedAt": "2026-07-19T16:15:36.863Z" }, "interactions": [ { @@ -21,7 +21,7 @@ "headers": { "content-type": "application/json" }, - "body": "{\"output\":{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":[{\"image\":\"https://dashscope-463f.oss-accelerate.aliyuncs.com/7d/7b/20260720/991d10e3/fe425a64-dc6f-4c67-a4fa-a06810984ba9.png?Expires=1785082919&OSSAccessKeyId=fixture&Signature=fixture\"}],\"role\":\"assistant\"}}]},\"usage\":{\"height\":512,\"image_count\":1,\"width\":512},\"request_id\":\"484490f4-9881-90eb-8d52-d270d2395920\"}" + "body": "{\"output\":{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":[{\"image\":\"https://dashscope-463f.oss-accelerate.aliyuncs.com/7d/04/20260720/991d10e3/93325a03-aa95-4dd3-a395-8d7246b5e72c.png?Expires=1785083536&OSSAccessKeyId=%5BREDACTED%5D&Signature=%5BREDACTED%5D\"}],\"role\":\"assistant\"}}]},\"usage\":{\"height\":512,\"image_count\":1,\"width\":512},\"request_id\":\"9f94d544-393b-9d6f-b005-4d0ff21f6fdf\"}" } } ] diff --git a/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-wan-2-7.json b/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-wan-2-7.json index 373482ec134e..37cf28a9f901 100644 --- a/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-wan-2-7.json +++ b/packages/ai/test/fixtures/recordings/alibaba-images/generates-with-wan-2-7.json @@ -3,7 +3,7 @@ "metadata": { "tags": ["prefix:alibaba-images", "provider:alibaba", "protocol:alibaba-images"], "name": "alibaba-images/generates-with-wan-2-7", - "recordedAt": "2026-07-19T16:05:23.617Z" + "recordedAt": "2026-07-19T16:15:40.689Z" }, "interactions": [ { @@ -21,7 +21,7 @@ "headers": { "content-type": "application/json" }, - "body": "{\"output\":{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":[{\"image\":\"https://dashscope-463f.oss-accelerate.aliyuncs.com/1d/ae/20260720/10739b93/1784477122_595381b5.png?Expires=1784563522&OSSAccessKeyId=fixture&Signature=fixture\",\"type\":\"image\"}],\"role\":\"assistant\"}}],\"finished\":true},\"usage\":{\"image_count\":1,\"input_tokens\":31,\"output_tokens\":2,\"size\":\"1024*1024\",\"total_tokens\":33},\"request_id\":\"ec5cacb5-c7b0-9e46-8459-49003a3816c3\"}" + "body": "{\"output\":{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":[{\"image\":\"https://dashscope-463f.oss-accelerate.aliyuncs.com/1d/0b/20260720/10739b93/1784477739_553aee1e.png?Expires=1784564140&OSSAccessKeyId=%5BREDACTED%5D&Signature=%5BREDACTED%5D\",\"type\":\"image\"}],\"role\":\"assistant\"}}],\"finished\":true},\"usage\":{\"image_count\":1,\"input_tokens\":31,\"output_tokens\":2,\"size\":\"1024*1024\",\"total_tokens\":33},\"request_id\":\"3eb93323-2d8f-9aa5-b010-82dd8fab80ea\"}" } } ] diff --git a/packages/ai/test/provider/alibaba-images.recorded.test.ts b/packages/ai/test/provider/alibaba-images.recorded.test.ts index 4980cd81b169..1058cc6554aa 100644 --- a/packages/ai/test/provider/alibaba-images.recorded.test.ts +++ b/packages/ai/test/provider/alibaba-images.recorded.test.ts @@ -1,5 +1,5 @@ -import { describe, expect } from "bun:test" -import { Effect } from "effect" +import { describe, expect, test } from "bun:test" +import { Effect, Option, Schema } from "effect" import { Image } from "../../src" import { Alibaba } from "../../src/providers" import { recordedTests } from "../recorded-test" @@ -8,11 +8,77 @@ const alibaba = Alibaba.configure({ apiKey: process.env.DASHSCOPE_API_KEY ?? "fixture", }) +const SIGNED_QUERY_CREDENTIALS = new Set( + [ + "AccessKeyId", + "AWSAccessKeyId", + "GoogleAccessId", + "OSSAccessKeyId", + "Signature", + "SecurityToken", + "X-Amz-Credential", + "X-Amz-Security-Token", + "X-Amz-Signature", + "X-Goog-Credential", + "X-Goog-Signature", + "X-OSS-Security-Token", + ].map((name) => name.replace(/[^a-z0-9]/gi, "").toLowerCase()), +) + +const redactSignedUrl = (value: string) => { + if (!URL.canParse(value)) return value + const url = new URL(value) + if (url.protocol !== "http:" && url.protocol !== "https:") return value + url.searchParams.forEach((_, name) => { + if (SIGNED_QUERY_CREDENTIALS.has(name.replace(/[^a-z0-9]/gi, "").toLowerCase())) + url.searchParams.set(name, "[REDACTED]") + }) + return url.toString() +} + +const redactSignedUrls = (body: string) => + Option.match(Schema.decodeUnknownOption(Schema.UnknownFromJsonString)(body), { + onNone: () => body, + onSome: (value) => + JSON.stringify( + (function redact(input: unknown): unknown { + if (typeof input === "string") return redactSignedUrl(input) + if (Array.isArray(input)) return input.map(redact) + if (input === null || typeof input !== "object") return input + return Object.fromEntries(Object.entries(input).map(([key, child]) => [key, redact(child)])) + })(value), + ), + }) + const recorded = recordedTests({ prefix: "alibaba-images", provider: "alibaba", protocol: "alibaba-images", requires: ["DASHSCOPE_API_KEY"], + options: { redact: { body: redactSignedUrls } }, +}) + +test("recorder redacts signed URL credentials in nested JSON strings repeatably", () => { + const body = JSON.stringify({ + output: { + image: + "https://example.oss.aliyuncs.com/image.png?Expires=1893456000&OSSAccessKeyId=access&Signature=signature&x-oss-security-token=token", + }, + mirrors: ["https://storage.googleapis.com/image.png?X-Goog-Credential=credential&X-Goog-Signature=signature"], + }) + const once = redactSignedUrls(body) + const twice = redactSignedUrls(once) + const parsed = JSON.parse(once) + const oss = new URL(parsed.output.image) + const gcs = new URL(parsed.mirrors[0]) + + expect(oss.searchParams.get("Expires")).toBe("1893456000") + expect(oss.searchParams.get("OSSAccessKeyId")).toBe("[REDACTED]") + expect(oss.searchParams.get("Signature")).toBe("[REDACTED]") + expect(oss.searchParams.get("x-oss-security-token")).toBe("[REDACTED]") + expect(gcs.searchParams.get("X-Goog-Credential")).toBe("[REDACTED]") + expect(gcs.searchParams.get("X-Goog-Signature")).toBe("[REDACTED]") + expect(twice).toBe(once) }) describe("Alibaba Images recorded", () => { diff --git a/packages/ai/test/provider/alibaba-images.test.ts b/packages/ai/test/provider/alibaba-images.test.ts index d8bfd12b93d3..b08da749706e 100644 --- a/packages/ai/test/provider/alibaba-images.test.ts +++ b/packages/ai/test/provider/alibaba-images.test.ts @@ -50,6 +50,7 @@ describe("Alibaba Images", () => { }) expect(result.image?.data).toContain("dashscope-result-intl") + expect(result.image?.expiresAt).toBe("2030-01-01T00:00:00.000Z") expect(result.image?.providerMetadata).toEqual({ alibaba: { modelId: "qwen-image-2.0-pro", @@ -193,4 +194,54 @@ describe("Alibaba Images", () => { ), ), ) + + it.effect("rejects parameters overlays owned by the protocol", () => + Image.generate({ + model: Alibaba.configure({ apiKey: "test" }).image("qwen-image-2.0"), + prompt: "A robot", + http: { body: { parameters: { watermark: true } } }, + }).pipe( + Effect.flip, + Effect.tap((error) => + Effect.sync(() => { + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("http.body cannot overlay protocol-owned field(s): parameters") + }), + ), + Effect.provide( + ImageClient.layer.pipe( + Layer.provide( + dynamicResponse(() => Effect.die("reserved body validation must happen before the request is sent")), + ), + ), + ), + ), + ) + + it.effect("ignores finite expiration values outside the Date range", () => + Effect.gen(function* () { + const result = yield* Image.generate({ + model: Alibaba.configure({ apiKey: "test" }).image("qwen-image-2.0"), + prompt: "A robot", + }) + + expect(result.image?.expiresAt).toBeUndefined() + expect(result.image?.providerMetadata?.alibaba?.expiresAt).toBeUndefined() + }).pipe( + Effect.provide( + ImageClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => { + const payload = response("qwen") + payload.output.choices[0].message.content[0].image = + "https://dashscope-result-intl.oss-cn-singapore.aliyuncs.com/result.png?Expires=9007199254740991" + return Effect.succeed( + input.respond(JSON.stringify(payload), { headers: { "content-type": "application/json" } }), + ) + }), + ), + ), + ), + ), + ) }) From bee140e63b1e047d83f855f51a97cbe423dc794d Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 22 Jul 2026 05:33:05 +0000 Subject: [PATCH 3/3] feat(ai): align Alibaba image API --- packages/ai/README.md | 24 ++ packages/ai/src/image.ts | 1 + packages/ai/src/protocols/alibaba-images.ts | 261 +++++------------ packages/ai/src/providers/alibaba.ts | 23 +- packages/ai/test/auth-options.types.ts | 8 +- packages/ai/test/exports.test.ts | 11 +- .../test/fixtures/images/edit-source-256.jpg | Bin 0 -> 2196 bytes .../edits-with-qwen-image-2-0.json | 32 +++ .../alibaba-images/edits-with-wan-2-7.json | 32 +++ packages/ai/test/image.types.ts | 22 +- .../provider/alibaba-images.recorded.test.ts | 59 +++- .../ai/test/provider/alibaba-images.test.ts | 264 +++++------------- 12 files changed, 321 insertions(+), 416 deletions(-) create mode 100644 packages/ai/test/fixtures/images/edit-source-256.jpg create mode 100644 packages/ai/test/fixtures/recordings/alibaba-images/edits-with-qwen-image-2-0.json create mode 100644 packages/ai/test/fixtures/recordings/alibaba-images/edits-with-wan-2-7.json diff --git a/packages/ai/README.md b/packages/ai/README.md index 6e4d87611cab..7f47809feae7 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -152,6 +152,30 @@ future string values and arbitrary native Gemini `generationConfig` fields remai their mapped aliases, and `http.body` is the final deep overlay. The selected model ID is sent to Gemini `generateContent` without a local allowlist. +Alibaba's international synchronous API uses the same request shape for Qwen and Wan image models: + +```ts +import { Alibaba } from "@opencode-ai/ai/providers" + +const response = + yield * + Image.generate({ + model: Alibaba.configure({ apiKey }).image("any-model-id"), + prompt: "Turn this source into a bright orange sun icon", + images: [ImageInput.bytes(sourceBytes, "image/jpeg")], + options: { + resolution: "2K", + promptExtend: false, + future_parameter: true, + }, + http, + }) +``` + +Known Qwen and Wan aliases autocomplete in one open request option object. Unknown native parameter keys pass +through, native keys override aliases, and `http.body.parameters` is the final deep overlay. Model IDs are not +locally allowlisted. Alibaba returns temporary signed URLs; persist each image before its optional `expiresAt`. + Z.ai image models infer open Z.ai-native options from the selected model: ```ts diff --git a/packages/ai/src/image.ts b/packages/ai/src/image.ts index ee3ab6202ad7..0c65e43c6f55 100644 --- a/packages/ai/src/image.ts +++ b/packages/ai/src/image.ts @@ -118,6 +118,7 @@ export type ImageRequestInput = Omit< export class GeneratedImage extends Schema.Class("Image.Generated")({ mediaType: Schema.String, data: Schema.Union([Schema.String, Schema.Uint8Array]), + expiresAt: Schema.optional(Schema.String), providerMetadata: Schema.optional(ProviderMetadata), }) {} diff --git a/packages/ai/src/protocols/alibaba-images.ts b/packages/ai/src/protocols/alibaba-images.ts index 23ae5d8b618f..a6e8cccabe5d 100644 --- a/packages/ai/src/protocols/alibaba-images.ts +++ b/packages/ai/src/protocols/alibaba-images.ts @@ -1,13 +1,6 @@ import { Effect, Schema } from "effect" import { Headers, HttpClientRequest } from "effect/unstable/http" -import { - GeneratedImage, - ImageModel, - ImageResponse, - type ImageModelDefaults, - type ImageRequest, - type ImageRoute, -} from "../image" +import { GeneratedImage, ImageModel, ImageResponse, type ImageRequestFor, type ImageRoute } from "../image" import { Auth, type Definition as AuthDefinition } from "../route/auth" import { InvalidProviderOutputReason, @@ -16,88 +9,45 @@ import { Usage, mergeHttpOptions, mergeJsonRecords, + type HttpOptions, } from "../schema" import { ProviderShared } from "./shared" +import { ImageInputs } from "./utils/image-input" const ADAPTER = "alibaba-images" export const DEFAULT_BASE_URL = "https://dashscope-intl.aliyuncs.com/api/v1" export const PATH = "/services/aigc/multimodal-generation/generation" -export type Family = "qwen" | "wan" - -export interface QwenImageOptions { - readonly negativePrompt?: string - readonly promptExtend?: boolean - readonly watermark?: boolean -} +export type AlibabaImageString = Known | (string & {}) -export interface WanColor { +export interface AlibabaColor { readonly hex: string readonly ratio: string } -export interface WanImageOptions { - readonly resolution?: "1K" | "2K" | "4K" +export type AlibabaImageOptions = { + readonly n?: number + readonly size?: string + readonly resolution?: AlibabaImageString<"1K" | "2K" | "4K"> + readonly negativePrompt?: string + readonly promptExtend?: boolean readonly thinkingMode?: boolean - readonly colorPalette?: ReadonlyArray + readonly colorPalette?: ReadonlyArray readonly watermark?: boolean -} - -export interface AlibabaImageOptions { - readonly qwen?: QwenImageOptions - readonly wan?: WanImageOptions -} - -declare module "../image" { - interface ImageProviderOptions { - readonly alibaba?: AlibabaImageOptions + readonly seed?: number +} & Record + +export type AlibabaImageBody = Record & { + readonly model: string + readonly input: { + readonly messages: ReadonlyArray<{ + readonly role: "user" + readonly content: ReadonlyArray<{ readonly image: string } | { readonly text: string }> + }> } + readonly parameters: Record } -const MessageInput = Schema.Struct({ - messages: Schema.Tuple([ - Schema.Struct({ - role: Schema.tag("user"), - content: Schema.Tuple([Schema.Struct({ text: Schema.String })]), - }), - ]), -}) - -const QwenBody = Schema.Struct({ - model: Schema.String, - input: MessageInput, - parameters: Schema.Struct({ - size: Schema.optional(Schema.String), - n: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(1))), - negative_prompt: Schema.optional(Schema.String), - prompt_extend: Schema.optional(Schema.Boolean), - watermark: Schema.optional(Schema.Boolean), - seed: Schema.optional(Schema.Int), - }), -}) -export type QwenBody = Schema.Schema.Type - -const WanBody = Schema.Struct({ - model: Schema.String, - input: MessageInput, - parameters: Schema.Struct({ - size: Schema.optional(Schema.String), - n: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(1))), - thinking_mode: Schema.optional(Schema.Boolean), - color_palette: Schema.optional( - Schema.Array( - Schema.Struct({ - hex: Schema.String, - ratio: Schema.String, - }), - ), - ), - watermark: Schema.optional(Schema.Boolean), - seed: Schema.optional(Schema.Int), - }), -}) -export type WanBody = Schema.Schema.Type - const AlibabaResponse = Schema.Struct({ output: Schema.optional( Schema.Struct({ @@ -136,60 +86,22 @@ const AlibabaResponse = Schema.Struct({ export interface ModelInput { readonly id: string - readonly family: Family readonly auth: AuthDefinition readonly baseURL?: string readonly headers?: Record - readonly defaults?: ImageModelDefaults -} - -const options = (request: ImageRequest): AlibabaImageOptions => ({ - ...request.model.defaults?.providerOptions?.alibaba, - ...request.providerOptions?.alibaba, - qwen: { - ...(request.model.defaults?.providerOptions?.alibaba?.qwen as QwenImageOptions | undefined), - ...(request.providerOptions?.alibaba?.qwen as QwenImageOptions | undefined), - }, - wan: { - ...(request.model.defaults?.providerOptions?.alibaba?.wan as WanImageOptions | undefined), - ...(request.providerOptions?.alibaba?.wan as WanImageOptions | undefined), - }, -}) - -const messageInput = (request: ImageRequest) => ({ - messages: [{ role: "user" as const, content: [{ text: request.prompt }] }] as const, -}) - -const qwenBody = (request: ImageRequest): QwenBody => { - const qwen = options(request).qwen - return { - model: request.model.id, - input: messageInput(request), - parameters: { - size: request.size === undefined ? undefined : `${request.size.width}*${request.size.height}`, - n: request.count, - negative_prompt: qwen?.negativePrompt, - prompt_extend: qwen?.promptExtend, - watermark: qwen?.watermark, - seed: request.seed, - }, - } + readonly http?: HttpOptions } -const wanBody = (request: ImageRequest): WanBody => { - const wan = options(request).wan +const nativeOptions = (options: AlibabaImageOptions | undefined) => { + if (!options) return undefined + const { resolution, negativePrompt, promptExtend, thinkingMode, colorPalette, ...native } = options return { - model: request.model.id, - input: messageInput(request), - parameters: { - size: - wan?.resolution ?? (request.size === undefined ? undefined : `${request.size.width}*${request.size.height}`), - n: request.count, - thinking_mode: wan?.thinkingMode, - color_palette: wan?.colorPalette === undefined ? undefined : [...wan.colorPalette], - watermark: wan?.watermark, - seed: request.seed, - }, + size: resolution, + negative_prompt: negativePrompt, + prompt_extend: promptExtend, + thinking_mode: thinkingMode, + color_palette: colorPalette, + ...native, } } @@ -221,21 +133,6 @@ const applyQuery = (url: string, query: Record | undefined) => { return next.toString() } -const PROTOCOL_BODY_FIELDS = new Set(["model", "input", "parameters"]) - -const bodyWithOverlay = Effect.fn("AlibabaImages.bodyWithOverlay")(function* ( - imageBody: QwenBody | WanBody, - overlay: Record | undefined, -) { - if (!overlay) return imageBody - const reserved = Object.keys(overlay).filter((key) => PROTOCOL_BODY_FIELDS.has(key)) - if (reserved.length > 0) - return yield* ProviderShared.invalidRequest( - `http.body cannot overlay protocol-owned field(s): ${reserved.join(", ")}`, - ) - return mergeJsonRecords(imageBody, overlay) ?? imageBody -}) - const expiration = (url: string) => { if (!URL.canParse(url)) return undefined const value = new URL(url).searchParams.get("Expires") @@ -245,32 +142,31 @@ const expiration = (url: string) => { } export const model = (input: ModelInput) => { - const route: ImageRoute = { - id: `${ADAPTER}-${input.family}`, - generate: Effect.fn("AlibabaImages.generate")(function* (request: ImageRequest, execute) { - if (request.aspectRatio !== undefined) - return yield* ProviderShared.invalidRequest("Alibaba Images does not support the common aspectRatio option") - if ( - input.family === "qwen" && - (request.model.defaults?.providerOptions?.alibaba?.wan !== undefined || - request.providerOptions?.alibaba?.wan !== undefined) - ) - return yield* ProviderShared.invalidRequest("Qwen Image does not accept providerOptions.alibaba.wan") - if ( - input.family === "wan" && - (request.model.defaults?.providerOptions?.alibaba?.qwen !== undefined || - request.providerOptions?.alibaba?.qwen !== undefined) - ) - return yield* ProviderShared.invalidRequest("Wan Image does not accept providerOptions.alibaba.qwen") - if (input.family === "wan" && request.size !== undefined && options(request).wan?.resolution !== undefined) - return yield* ProviderShared.invalidRequest("Wan Image accepts either size or resolution, not both") - - const requestBody = yield* ProviderShared.validateWith( - Schema.decodeUnknownEffect(input.family === "qwen" ? QwenBody : WanBody), - )(input.family === "qwen" ? qwenBody(request) : wanBody(request)) - const http = mergeHttpOptions(request.model.defaults?.http, request.http) - const overlaidBody = yield* bodyWithOverlay(requestBody, http?.body) - const text = ProviderShared.encodeJson(overlaidBody) + const route: ImageRoute = { + id: ADAPTER, + generate: Effect.fn("AlibabaImages.generate")(function* (request: ImageRequestFor, execute) { + const content = yield* Effect.forEach(request.images ?? [], (image) => { + if (image.type === "bytes") return Effect.succeed({ image: ImageInputs.dataUrl(image) }) + if (image.type === "url") return Effect.succeed({ image: image.url }) + return ImageInputs.invalid(ADAPTER, "Alibaba Images accepts image URLs, data URLs, and bytes") + }) + const requestBody: AlibabaImageBody = { + model: request.model.id, + input: { messages: [{ role: "user", content: [...content, { text: request.prompt }] }] }, + parameters: nativeOptions(request.options) ?? {}, + } + const http = mergeHttpOptions(request.model.http, request.http) + const overlay = mergeJsonRecords(requestBody, http?.body) ?? requestBody + const body: AlibabaImageBody = { + ...overlay, + model: requestBody.model, + input: requestBody.input, + parameters: + overlay.parameters !== null && typeof overlay.parameters === "object" && !Array.isArray(overlay.parameters) + ? (overlay.parameters as Record) + : requestBody.parameters, + } + const text = ProviderShared.encodeJson(body) const url = applyQuery(`${(input.baseURL ?? DEFAULT_BASE_URL).replace(/\/$/, "")}${PATH}`, http?.query) const headers = yield* Auth.toEffect(input.auth)({ request, @@ -293,26 +189,20 @@ export const model = (input: ModelInput) => { ) if (decoded.code !== undefined || decoded.output === undefined) return yield* providerError(decoded.code, decoded.message, decoded.request_id) - const urls = decoded.output.choices.flatMap((choice) => choice.message.content.map((content) => content.image)) + const urls = decoded.output.choices.flatMap((choice) => choice.message.content.map((item) => item.image)) if (urls.length === 0) return yield* invalidOutput("Alibaba Images returned no images", { requestId: decoded.request_id }) return new ImageResponse({ - images: urls.map((url) => { - const expiresAt = expiration(url) - return new GeneratedImage({ - mediaType: "image/png", - data: url, - expiresAt, - providerMetadata: { - alibaba: { - modelId: request.model.id, - family: input.family, - expiresAt, - }, - }, - }) - }), + images: urls.map( + (url) => + new GeneratedImage({ + mediaType: "image/png", + data: url, + expiresAt: expiration(url), + providerMetadata: { alibaba: { modelId: request.model.id } }, + }), + ), usage: decoded.usage === undefined ? undefined @@ -322,17 +212,16 @@ export const model = (input: ModelInput) => { totalTokens: decoded.usage.total_tokens, providerMetadata: { alibaba: decoded.usage }, }), - providerMetadata: { - alibaba: { - requestId: decoded.request_id, - modelId: request.model.id, - family: input.family, - }, - }, + providerMetadata: { alibaba: { requestId: decoded.request_id, modelId: request.model.id } }, }) }), } - return ImageModel.make({ id: input.id, provider: "alibaba", route, defaults: input.defaults }) + return ImageModel.make({ + id: input.id, + provider: "alibaba", + route, + http: input.http, + }) } export const AlibabaImages = { diff --git a/packages/ai/src/providers/alibaba.ts b/packages/ai/src/providers/alibaba.ts index b4cfddfb87f2..19f95c44e9d7 100644 --- a/packages/ai/src/providers/alibaba.ts +++ b/packages/ai/src/providers/alibaba.ts @@ -1,11 +1,8 @@ -import type { ImageModel } from "../image" -import { AlibabaImages, type AlibabaImageOptions, type Family } from "../protocols/alibaba-images" +import { AlibabaImages } from "../protocols/alibaba-images" import { AuthOptions, type ProviderAuthOption } from "../route/auth-options" -import { HttpOptions, ProviderID, mergeHttpOptions } from "../schema" +import { HttpOptions, ProviderID } from "../schema" -export type { AlibabaImageOptions, QwenImageOptions, WanColor, WanImageOptions } from "../protocols/alibaba-images" - -export type AlibabaImageModelID = "qwen-image-2.0" | "qwen-image-2.0-pro" | "wan2.7-image" | "wan2.7-image-pro" +export type { AlibabaColor, AlibabaImageOptions } from "../protocols/alibaba-images" export const id = ProviderID.make("alibaba") @@ -13,28 +10,18 @@ export type Config = ProviderAuthOption<"optional"> & { readonly baseURL?: string readonly headers?: Record readonly http?: HttpOptions.Input - readonly image?: { - readonly providerOptions?: AlibabaImageOptions - } } const auth = (options: ProviderAuthOption<"optional">) => AuthOptions.bearer(options, "DASHSCOPE_API_KEY") -const family = (modelID: AlibabaImageModelID): Family => (modelID.startsWith("qwen-") ? "qwen" : "wan") - export const configure = (input: Config = {}) => { - const image = (modelID: AlibabaImageModelID): ImageModel => + const image = (modelID: string) => AlibabaImages.model({ id: modelID, - family: family(modelID), auth: auth(input), baseURL: input.baseURL, headers: input.headers, - defaults: { - providerOptions: - input.image?.providerOptions === undefined ? undefined : { alibaba: { ...input.image.providerOptions } }, - http: mergeHttpOptions(input.http === undefined ? undefined : HttpOptions.make(input.http)), - }, + http: input.http === undefined ? undefined : HttpOptions.make(input.http), }) return { diff --git a/packages/ai/test/auth-options.types.ts b/packages/ai/test/auth-options.types.ts index 2a7e9626c982..4f0b3a995423 100644 --- a/packages/ai/test/auth-options.types.ts +++ b/packages/ai/test/auth-options.types.ts @@ -252,8 +252,6 @@ GitHubCopilot.configure({ baseURL: "https://copilot.test", apiKey: "copilot-key" // @ts-expect-error GitHub Copilot model selectors only accept model ids. GitHubCopilot.configure({ baseURL: "https://copilot.test", apiKey: "copilot-key" }).model("gpt-4.1", {}) -Alibaba.configure({ image: { providerOptions: { qwen: { promptExtend: true } } } }).image("qwen-image-2.0") -// @ts-expect-error Alibaba Qwen options are closed and reject unimplemented parameters. -Alibaba.configure({ image: { providerOptions: { qwen: { style: "photorealistic" } } } }) -// @ts-expect-error Alibaba Wan options are closed and reject unimplemented parameters. -Alibaba.configure({ image: { providerOptions: { wan: { parameters: { watermark: true } } } } }) +Alibaba.configure({ apiKey: "dashscope-key" }).image("any-model-id") +// @ts-expect-error Alibaba image options are request-scoped, not provider configuration. +Alibaba.configure({ image: { options: { promptExtend: true } } }) diff --git a/packages/ai/test/exports.test.ts b/packages/ai/test/exports.test.ts index 4629100da514..b0337e8cb21a 100644 --- a/packages/ai/test/exports.test.ts +++ b/packages/ai/test/exports.test.ts @@ -3,6 +3,7 @@ import { ImageInput, LLM, LLMClient, Provider } from "@opencode-ai/ai" import { Route, Protocol } from "@opencode-ai/ai/route" import { Provider as ProviderSubpath } from "@opencode-ai/ai/provider" import { + Alibaba, CloudflareAIGateway, CloudflareWorkersAI, OpenAI, @@ -11,7 +12,13 @@ import { XAI, } from "@opencode-ai/ai/providers" import * as GitHubCopilot from "@opencode-ai/ai/providers/github-copilot" -import { OpenAIChat, OpenAICompatibleChat, OpenAICompatibleResponses, OpenAIResponses } from "@opencode-ai/ai/protocols" +import { + AlibabaImages, + OpenAIChat, + OpenAICompatibleChat, + OpenAICompatibleResponses, + OpenAIResponses, +} from "@opencode-ai/ai/protocols" import * as AnthropicMessages from "@opencode-ai/ai/protocols/anthropic-messages" describe("public exports", () => { @@ -32,6 +39,7 @@ describe("public exports", () => { test("provider barrels expose user-facing facades", async () => { const { OpenAICompatibleResponses } = await import("@opencode-ai/ai/providers") + expect(Alibaba.configure({ apiKey: "fixture" }).image("any-model-id").provider).toBe("alibaba") expect(OpenAI.model).toBeFunction() expect(OpenAI.provider.responses).toBe(OpenAI.responses) expect(OpenAI.provider.responsesWebSocket).toBe(OpenAI.responsesWebSocket) @@ -72,6 +80,7 @@ describe("public exports", () => { }) test("protocol barrels expose supported low-level routes", () => { + expect(AlibabaImages.DEFAULT_BASE_URL).toBe("https://dashscope-intl.aliyuncs.com/api/v1") expect(OpenAIChat.route.id).toBe("openai-chat") expect(OpenAICompatibleChat.route.id).toBe("openai-compatible-chat") expect(OpenAICompatibleResponses.route.id).toBe("openai-compatible-responses") diff --git a/packages/ai/test/fixtures/images/edit-source-256.jpg b/packages/ai/test/fixtures/images/edit-source-256.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d5269b3d15f5d302cba51eeecac62c06d1e278f4 GIT binary patch literal 2196 zcmcIkiC2@?68|8TrH=p~i&7Brfg3K=7}7OGwy3NLWH%st0TTfX+SV&Ye4Fe)F5T_l&Av z^#(BX_WG9>prHu>8tMh8wg5w)P<)ufaXUE74tD$mpb`TffR2uiu8!_M-V0O@`lsmY z>w_Su-l0%^eM3VKG%_|)Ux3EOCMG6kX1jLn-fd}ZZL8t}L^V7FfM}?bY>Sq*jwVE7 z$7i43eefBo$!Kb7X=!O|@A%-=(1d&j8vPUY^$x8Y?%ICF-?(>`eHXiVO zrrf8a@fQ=y;}&+F%mY8k@c~cu3;=chC=jAy4QPS@1O!xrfF49cZK%&kpzYZ|^NPeORvhI%_hvA>pa-5GT=z>!SeZgw zr|9A25$|y#EW?splDeZB*YE5kuxZ#uml?{!Wp<8DB~0$PE>ci=`vINd^Y9L2OeZ)-;-$=Ha<|gNICraS_=H<7%M#Ija|I zYR{uO>zR7Kqf>lk8S2SLz_V)y$!QV=yX4G-sPb)Uv14TA(j00~euneZnieJUYEc1m ztDhIG9k2;k0TpxtV$8U`VLz=}6i0D&E0!|{JYh`Avp@0Fz!6)6uwJ+H^bue3>^n?L z8z!4}qACjCj23lBsa<9D1{wGL;T7gzedOTPGMa*i-O0WhwjiE7kXq`J@mF#<;NiPC@W~jKD;#cPBbYP$b8{mRoF{1ofms)*}QxjEL&8T z=J*!N`~pykklWiq1_WskC(qA5-m*#VSFttYU~Ni5yp59E4YxmujR(nXts~Tl)v9*? zkrjyNcF-T1vElhf{X2HZ4vFGOQ2i3mA_}_?)tSWf#GQ;p_4JHkVS%xM4JFJeIZT3B zKyIplo!rp%`-%{&0ag^PX@x&17c_u|+nBmSPpnl>bu;YE@{454CRuJZr=nu@%aPjy zSU+Z}kF@i(5k2(nYRtV&`$$P|_yl3P-W9dVXGi6nycA3_Bi*Ir`GjTdm~A|@)~88@ zvf=c2!QsB{62wJh$Z*J>VRjb=SA2z>K~m~+c~=kTN}R+f{=Y*%pB-Xd5I36J^DwNe zq0)Z71A^)v-Oc?0R**(GyQ~8Cc6{+Lw(y%yCDPPcQc8q^^ojLDibzVg_?l20(9<+_ zSlZ|*zDSOqQFQFL0gICnC>S5$v2 zo=qhn#|?%CPkC5;F)X5@Qc_nMMVG+XhPA zSEXy!|NJa1vhJvW;wA@(*&-kPc9Ro$$BvJdO9xbd<*T{Zr7bqN{h0F9yy0EZz4)wp zbcAu8J@0ox9B0h(K@4g?0^R#YMR8@Q2)OH)`gJ}%~{l7T1ht98_# zVcb}d{+H`7P2_DZ?z}F3)E;1Qz%8AW`C{O=-q9ecd*Kkpzb;%oWL(A#-Ow{~jjjPK uW0A3@0*rFozI8L>UZ9RLjuq!07(usmB_cS#%D-{`=`TI~+YAHM!2ba(51JYP literal 0 HcmV?d00001 diff --git a/packages/ai/test/fixtures/recordings/alibaba-images/edits-with-qwen-image-2-0.json b/packages/ai/test/fixtures/recordings/alibaba-images/edits-with-qwen-image-2-0.json new file mode 100644 index 000000000000..02aa4e241174 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/alibaba-images/edits-with-qwen-image-2-0.json @@ -0,0 +1,32 @@ +{ + "version": 1, + "metadata": { + "tags": [ + "prefix:alibaba-images", + "provider:alibaba", + "protocol:alibaba-images" + ], + "name": "alibaba-images/edits-with-qwen-image-2-0", + "recordedAt": "2026-07-22T05:30:23.625Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"qwen-image-2.0\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":[{\"image\":\"data:image/jpeg;base64,/9j/4AAQSkZJRgABAgAAAQABAAD//gAQTGF2YzYwLjMxLjEwMgD/2wBDAAgQEBMQExYWFhYWFhoYGhsbGxoaGhobGxsdHR0iIiIdHR0bGx0dICAiIiUmJSMjIiMmJigoKDAwLi44ODpFRVP/xABrAAEBAQEBAQAAAAAAAAAAAAAABwgGBAUBAQAAAAAAAAAAAAAAAAAAAAAQAAICAQEFBwMEAwEAAAAAAAABAwIEEQYFEiFhwXGBUUExE6GRU0LwciLh8bEjEQEAAAAAAAAAAAAAAAAAAAAA/8AAEQgAgACAAwEiAAIRAAMRAP/aAAwDAQACEQMRAD8Av4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB458iLFjtLNesdK+9rPRf76e4yMiPFivNLbhpHV2s+i7X6LzMhb33xPvabitrWKrfxxa8qrzfnZ+r8EBTc/bPm64USa/LLrz61omvq/AnMu0W9ZXq8q9elFSi+ldTigB2se0W9Ynqsq9ul1S6+tSiYG2b1Vc2Jafli15dXRt/R+BBgBuvHyIcqOssN6yUt7Wq9V/h9HzPYY03RvefdM3HTW0dn/AOkWvKy815WXo/ua+x8iPKhpNFbipJVWq/36r2a8wPaAAAAAAAAAAIDtnnvWLCq+WnyydfSlX9X9iDHZ7QyuXeuW3+m6ou6lUjjAAAAAAAXbYzeDVpcKz5NfLH0ftdLv5P7kJOw2flcW9cRr1k4H3XTr2gbHAAAAAAAAAAGO9oYnFvXLT/VdXXdeqZxhdts8Bq0WZVcmvik6NaujffzX2ISAAAAAADr9wRuXeuGl6ScfhRO3YcgXHYzAdpZcyy/rRfFH1s+dmu5aLxA0MAAAAAAAAAAPDk40WXDJDLXipJXha7V1Xuupj7eu6p91TuORO1Hr8cmnK67LL1Rs8+fk4sOZFaKeOslLe6f/AFP3T8muYGGAXHP2Nkq3bDlV6/jl5WXRXXJ+KROpdxb0ielsOZ/wXGvvRsDkwdXHuLekr0WHMv5V4F97tFDwNjZrtWzJVHX8cb4rvo7acK8NQJtuvdc+9Z1FGtKrT5JNP60r2vyXqbBxMWLCgjgiXDSNaLzfm31b5sYuJBhRKKCOsdF6L1fm37t9WfRAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/9k=\"},{\"text\":\"Turn the black source shape into a bright orange sun icon on a pale blue background.\"}]}]},\"parameters\":{\"prompt_extend\":false,\"watermark\":false}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/json" + }, + "body": "{\"output\":{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":[{\"image\":\"https://dashscope-463f.oss-accelerate.aliyuncs.com/7d/89/20260722/74716928/0e082b32-9ff8-4e81-aa78-bde6b41d33ae.png?Expires=1785304023&OSSAccessKeyId=%5BREDACTED%5D&Signature=%5BREDACTED%5D\"}],\"role\":\"assistant\"}}]},\"usage\":{\"height\":1024,\"image_count\":1,\"width\":1024},\"request_id\":\"13d119d7-dbf1-98a7-9fae-d4ad6441f8a7\"}" + } + } + ] +} diff --git a/packages/ai/test/fixtures/recordings/alibaba-images/edits-with-wan-2-7.json b/packages/ai/test/fixtures/recordings/alibaba-images/edits-with-wan-2-7.json new file mode 100644 index 000000000000..c6cd5a71ecd0 --- /dev/null +++ b/packages/ai/test/fixtures/recordings/alibaba-images/edits-with-wan-2-7.json @@ -0,0 +1,32 @@ +{ + "version": 1, + "metadata": { + "tags": [ + "prefix:alibaba-images", + "provider:alibaba", + "protocol:alibaba-images" + ], + "name": "alibaba-images/edits-with-wan-2-7", + "recordedAt": "2026-07-22T05:30:36.133Z" + }, + "interactions": [ + { + "transport": "http", + "request": { + "method": "POST", + "url": "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation", + "headers": { + "content-type": "application/json" + }, + "body": "{\"model\":\"wan2.7-image\",\"input\":{\"messages\":[{\"role\":\"user\",\"content\":[{\"image\":\"data:image/jpeg;base64,/9j/4AAQSkZJRgABAgAAAQABAAD//gAQTGF2YzYwLjMxLjEwMgD/2wBDAAgICAkICQsLCwsLCw0MDQ0NDQ0NDQ0NDQ0ODg4REREODg4NDQ4OEBARERITEhERERETExQUFBgYFxccHB0iIin/xAB+AAEAAwADAQEAAAAAAAAAAAAABgcIAgMBBQQBAQAAAAAAAAAAAAAAAAAAAAAQAQABAgIGBgYHBwUBAQAAAAABAgMEERIhMUEFBoFCB1ETUhRhoXEyIpHBglMjQzMkFcSx4XNidKJy8BY0khEBAAAAAAAAAAAAAAAAAAAAAP/AABEIAQABAAMBIgACEQADEQD/2gAMAwEAAhEDEQA/AL/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABx264+l7u2urLOrPf5f6opzNzhw3lqn8erO75Mrv+HWps3KdlcSCWzOXzTOWW7ahnGOf+AcImabmK0rkfl+FiY8vWjD1RsqzZ15g594xzBq0vCtb7eVivydbwLdW2jNCtUa6Z192X1yC+sb2zxMfs2A1/wCo93nwfvR+vtb47XPyWdCP+eHq/nhFS7P8fa4zETtnPoyBbsdrfHbc/NZ0vt4eP5YR93B9s8RH7TgM5/1GXf5MH7lDRFMbJ9jydYNfcH7QeA8YnRt4rRr+78LEz5utOHojZTmmtMxVGlTrz6PVvYN+WrXn0f1TPgHPXF+XtVNfiW/u8rFPn602Lk7a8wbB2xqnp/o9iOmUO5Z5z4bzLH4NWhd+7yu1efr1WbdPw0TPsTCNuuMpndtBzAAAAAAAAAAAAAAAAABwz+XOdRTM56+g1U51d+SH848zRy1wub8/rzl4ce67bpq1+Fcp+GvfAPi88890ct0+j4adPF17I106GjNqrr2Lluc6K536vey/icTex9+q9fq0q5y06sqY2UxFOqmKY2REaodd7E38XeqvX6866stKvKmNlOUfLTERsiI1Q/PFuJiZ/wDzHm79e7IHIAAAAAAAH6LF+9g8TTdsVZV055VZUznnTMTqqiY2TMbGneROe7fMFv0bEzoYqnZGU1aWc3aupYotxlRRG/2ssxTET7t39XfavXsPc8azVlXHXyp1Zxo/DVExsmY2A3bVMxGfdtj3k1atJD+S+Z6eZuGRd/Ot/qx/yu3Yp/Kt0/DRuj3phnnrjbu+sHYAAAAAAAAAAAAAAADqmqKfi1d3/YZA564/+/8AjF2uP0qNDw/tWbUVfl0VbaN/Q0X2gcY/dHA71ynVcq0NDov2Yq20V07Kt7IMT+Hozv2fTrAAAAAAAAAAAABL+SePf+f4tZxH5den4n2bN2mn8u5O2vdHvbEiqmrXRrz/AO72Cpn5JiNm/wClrns64xPFuCWqq9dynT0+m/ey2UUxsp3AsAAAAAAAAAAAAAAAAGf+2jHRPoWF30+P7fRavL9aiqoyqWp2u1TXx+mjy5+3D4aVUzOlGkAAAAAAAAAAAABGvOF5djGOim9j8Nvr9Hy+zTiqvL9ai6p0aplaXZLVNvmK3TuuaX+3DYgGpwAAAAAAAAAAAAAAAZe7XaZo4/TX5s/Zh8NCqZjRjRXv20YGI9CxW+rx/Z6LT5vqUVVOdQOIAAAAAAAAAAAPe+lZvZJRNXMduvdRpf7sNiIVjTriavcu3sWwcXMRj7++36Pl9qnFU9/1A0UAAAAAAAAAAAAAAACv+0Xg88W4Beoo13KNDQ6cRZmdtdNOyneyRV80aU7NzeNdMVZxVricsuj3Mec68D/8/wAWxGH6lfheH9mzaqq69c7a98+4ERAAAAAAAAAAAAnVRV0fzau7MeDzw3gduuqMq72lpfYv34jZXVGyfUzpytwSrmHi9nCU/DVp59Fm5Xvrtz1O9s61aos0aFuNGI2RrnbOe/MHcAAAAAAAAAAAAAAADqrp1erehXPPKtPMnDpiP17eXh7etctaX5tuj4aN/QnExm8qjOc4207OkGEK7d2zXNu5GjXGWrOJ3Z7YzjY6o0JtzMRlMZd/e0vz/wAgxxmmcZhIyv0/FTnnnn4NEa7l+ij4aZ3M23rV21XNuuMq6cs9cb4zjZq2A6gAAAAAAAe6VFVMURHTr97lTRVeqiiiM58ucR69s5e8imquqm3bjOZz0de3LXO1o3s/5A/d2jxDGxne16NOeX31uddrEVU/DMTrpBIez3lOOXcDp3f173x+rw7l6I+G7co+GvdksKnVM0xsj63u2fXD3Pf9IOQAAAAAAAAAAAAAAAAAOGqde/dKCc28h4DmWjSj8DEfe/iXN9vqePbo+GjLpz2p1FEdG4mO/wCbujYDGPHOVOL8vVaOLs5U9+nZ/wAd1u7cnrwjXyVaon2S3hdt0XaJt1RnE7Y1xsnPcr3jHZpwTiU6VNHg1z1tLEXPLunE0xsp9oMpzlS4xGl18uhdGM7GcbajPDYzxZ/sW6O7z4yfWj1zsr5oidWH04/uYSn+KBXEzRGyrS6Jh5lprJo7KeaM/wD5tCP7uEq/in3sJ2OcSux+0YrwvV4Nqvv8mM9wKZ0qqNVWr6J/kkPBuWeL8eq0cNazp79OzHmnr3aPJLQ3B+yvgfDp0sRT6XXG/PEWfNuoxVUbJj6FjYfD2cLR4VmjQpjdnVO2ZnbVMzvkED5U7PcBy9ldr/Gv+f8AEoy/Uj4YxFyj4a8tm5Ym31S8y3U6vaT3SDmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD//Z\"},{\"text\":\"Turn the black source shape into a bright orange sun icon on a pale blue background.\"}]}]},\"parameters\":{\"size\":\"1K\",\"thinking_mode\":false,\"watermark\":false}}" + }, + "response": { + "status": 200, + "headers": { + "content-type": "application/json" + }, + "body": "{\"output\":{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":[{\"image\":\"https://dashscope-463f.oss-accelerate.aliyuncs.com/1d/d0/20260722/10739b93/1784698235_08b3b146.png?Expires=1784784635&OSSAccessKeyId=%5BREDACTED%5D&Signature=%5BREDACTED%5D\",\"type\":\"image\"}],\"role\":\"assistant\"}}],\"finished\":true},\"usage\":{\"image_count\":1,\"input_tokens\":9430,\"output_tokens\":2,\"size\":\"1024*1024\",\"total_tokens\":9432},\"request_id\":\"e55e017f-f02a-92f3-8405-354906e1159a\"}" + } + } + ] +} diff --git a/packages/ai/test/image.types.ts b/packages/ai/test/image.types.ts index c6274963949f..dc1f44609ded 100644 --- a/packages/ai/test/image.types.ts +++ b/packages/ai/test/image.types.ts @@ -7,7 +7,7 @@ import { type ImageRequestFor, type ImageRoute, } from "../src" -import { Google, OpenAI, XAI, ZAI } from "../src/providers" +import { Alibaba, Google, OpenAI, XAI, ZAI } from "../src/providers" type GoogleLikeOptions = { readonly aspectRatio?: "1:1" | "16:9" @@ -58,6 +58,26 @@ Image.generate({ model: googleProvider, prompt: "A lighthouse", options: { seed: // @ts-expect-error Known Google boolean options retain their value kind. Image.generate({ model: googleProvider, prompt: "A lighthouse", options: { includeThoughts: "yes" } }) +const alibaba = Alibaba.configure({ apiKey: "test" }).image("any-model-id") +Image.generate({ + model: alibaba, + prompt: "A lighthouse", + images: [ImageInput.url("https://example.com/source.png")], + options: { + resolution: "2K", + negativePrompt: "fog", + promptExtend: true, + thinkingMode: false, + colorPalette: [{ hex: "#112233", ratio: "100.00%" }], + watermark: false, + future_parameter: true, + }, +}) +// @ts-expect-error Known Alibaba boolean options retain their value kind. +Image.generate({ model: alibaba, prompt: "A lighthouse", options: { promptExtend: "yes" } }) +// @ts-expect-error Known Alibaba numeric options retain their value kind. +Image.generate({ model: alibaba, prompt: "A lighthouse", options: { seed: "42" } }) + const openai = OpenAI.image("gpt-image-2") // @ts-expect-error Image generation options are request-scoped, not provider configuration. OpenAI.configure({ image: { options: { quality: "medium" } } }) diff --git a/packages/ai/test/provider/alibaba-images.recorded.test.ts b/packages/ai/test/provider/alibaba-images.recorded.test.ts index 1058cc6554aa..12778b62aa8e 100644 --- a/packages/ai/test/provider/alibaba-images.recorded.test.ts +++ b/packages/ai/test/provider/alibaba-images.recorded.test.ts @@ -1,7 +1,8 @@ import { describe, expect, test } from "bun:test" import { Effect, Option, Schema } from "effect" -import { Image } from "../../src" +import { Image, ImageInput } from "../../src" import { Alibaba } from "../../src/providers" +import { dimensions } from "../lib/image" import { recordedTests } from "../recorded-test" const alibaba = Alibaba.configure({ @@ -58,6 +59,15 @@ const recorded = recordedTests({ options: { redact: { body: redactSignedUrls } }, }) +const inspectLive = async (value: string | Uint8Array | undefined) => { + if (process.env.RECORD !== "true" || typeof value !== "string") return + const response = await fetch(value) + const bytes = new Uint8Array(await response.arrayBuffer()) + expect(response.headers.get("content-type")).toMatch(/^image\//) + expect(dimensions(bytes).width).toBeGreaterThan(0) + expect(dimensions(bytes).height).toBeGreaterThan(0) +} + test("recorder redacts signed URL credentials in nested JSON strings repeatably", () => { const body = JSON.stringify({ output: { @@ -87,8 +97,7 @@ describe("Alibaba Images recorded", () => { const response = yield* Image.generate({ model: alibaba.image("qwen-image-2.0"), prompt: "A simple flat black circle centered on a plain white background.", - size: { width: 512, height: 512 }, - providerOptions: { alibaba: { qwen: { promptExtend: false, watermark: false } } }, + options: { size: "512*512", promptExtend: false, watermark: false }, }) expect(response.images).toHaveLength(1) @@ -102,12 +111,54 @@ describe("Alibaba Images recorded", () => { const response = yield* Image.generate({ model: alibaba.image("wan2.7-image"), prompt: "A simple flat black square centered on a plain white background.", - providerOptions: { alibaba: { wan: { resolution: "1K", thinkingMode: false, watermark: false } } }, + options: { resolution: "1K", thinkingMode: false, watermark: false }, + }) + + expect(response.images).toHaveLength(1) + expect(response.image?.mediaType).toBe("image/png") + expect(response.image?.data).toStartWith("https://") + }), + ) + + recorded.effect("edits with Qwen Image 2.0", () => + Effect.gen(function* () { + const response = yield* Image.generate({ + model: alibaba.image("qwen-image-2.0"), + prompt: "Turn the black source shape into a bright orange sun icon on a pale blue background.", + images: [ + ImageInput.bytes( + yield* Effect.promise(() => Bun.file("test/fixtures/images/edit-source.jpg").bytes()), + "image/jpeg", + ), + ], + options: { promptExtend: false, watermark: false }, + }) + + expect(response.images).toHaveLength(1) + expect(response.image?.mediaType).toBe("image/png") + expect(response.image?.data).toStartWith("https://") + yield* Effect.promise(() => inspectLive(response.image?.data)) + }), + ) + + recorded.effect("edits with Wan 2.7", () => + Effect.gen(function* () { + const response = yield* Image.generate({ + model: alibaba.image("wan2.7-image"), + prompt: "Turn the black source shape into a bright orange sun icon on a pale blue background.", + images: [ + ImageInput.bytes( + yield* Effect.promise(() => Bun.file("test/fixtures/images/edit-source-256.jpg").bytes()), + "image/jpeg", + ), + ], + options: { resolution: "1K", thinkingMode: false, watermark: false }, }) expect(response.images).toHaveLength(1) expect(response.image?.mediaType).toBe("image/png") expect(response.image?.data).toStartWith("https://") + yield* Effect.promise(() => inspectLive(response.image?.data)) }), ) }) diff --git a/packages/ai/test/provider/alibaba-images.test.ts b/packages/ai/test/provider/alibaba-images.test.ts index b08da749706e..17f54bd023f3 100644 --- a/packages/ai/test/provider/alibaba-images.test.ts +++ b/packages/ai/test/provider/alibaba-images.test.ts @@ -1,247 +1,109 @@ import { describe, expect } from "bun:test" import { Effect, Layer } from "effect" -import { HttpClientRequest } from "effect/unstable/http" -import { Image, ImageClient } from "../../src" +import { Image, ImageClient, ImageInput } from "../../src" import { Alibaba } from "../../src/providers" import { it } from "../lib/effect" import { dynamicResponse } from "../lib/http" -const response = (family: "qwen" | "wan") => ({ - output: { - choices: [ - { - finish_reason: "stop", - message: { - role: "assistant", - content: [ - { - image: "https://dashscope-result-intl.oss-cn-singapore.aliyuncs.com/result.png?Expires=1893456000", - ...(family === "wan" ? { type: "image" } : {}), - }, - ], - }, - }, - ], - ...(family === "wan" ? { finished: true } : {}), - }, - usage: - family === "wan" - ? { image_count: 1, input_tokens: 10, output_tokens: 2, total_tokens: 12, size: "2048*2048" } - : { image_count: 1, width: 1024, height: 768 }, - request_id: `request-${family}`, +const payload = (url = "https://example.com/result.png?Expires=1893456000") => ({ + output: { choices: [{ message: { content: [{ image: url }] } }] }, + usage: { total_tokens: 12 }, + request_id: "request-alibaba", }) -describe("Alibaba Images", () => { - it.effect("generates Qwen Image 2.0 through the international synchronous route", () => - Effect.gen(function* () { - const result = yield* Image.generate({ - model: Alibaba.configure({ - apiKey: "test", - baseURL: "https://dashscope-intl.test/api/v1", - image: { providerOptions: { qwen: { promptExtend: true, watermark: false } } }, - http: { headers: { "x-default": "yes" } }, - }).image("qwen-image-2.0-pro"), - prompt: "A robot tending a rooftop garden", - count: 2, - size: { width: 1024, height: 768 }, - seed: 42, - providerOptions: { alibaba: { qwen: { negativePrompt: "blurry" } } }, - http: { headers: { "x-request": "yes" }, query: { trace: "1" }, body: { metadata: "test" } }, - }) - - expect(result.image?.data).toContain("dashscope-result-intl") - expect(result.image?.expiresAt).toBe("2030-01-01T00:00:00.000Z") - expect(result.image?.providerMetadata).toEqual({ - alibaba: { - modelId: "qwen-image-2.0-pro", - family: "qwen", - expiresAt: "2030-01-01T00:00:00.000Z", - }, - }) - expect(result.providerMetadata).toEqual({ - alibaba: { requestId: "request-qwen", modelId: "qwen-image-2.0-pro", family: "qwen" }, - }) - }).pipe( - Effect.provide( - ImageClient.layer.pipe( - Layer.provide( - dynamicResponse((input) => - Effect.gen(function* () { - const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie) - expect(request.url).toBe( - "https://dashscope-intl.test/api/v1/services/aigc/multimodal-generation/generation?trace=1", - ) - expect(request.headers.get("authorization")).toBe("Bearer test") - expect(request.headers.get("x-default")).toBe("yes") - expect(request.headers.get("x-request")).toBe("yes") - expect(JSON.parse(input.text)).toEqual({ - model: "qwen-image-2.0-pro", - input: { - messages: [{ role: "user", content: [{ text: "A robot tending a rooftop garden" }] }], - }, - parameters: { - size: "1024*768", - n: 2, - negative_prompt: "blurry", - prompt_extend: true, - watermark: false, - seed: 42, - }, - metadata: "test", - }) - return input.respond(JSON.stringify(response("qwen")), { - headers: { "content-type": "application/json" }, - }) - }), - ), - ), - ), - ), +const layer = (inspect: (body: Record) => void, url?: string) => + ImageClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => { + inspect(JSON.parse(input.text)) + return Effect.succeed( + input.respond(JSON.stringify(payload(url)), { headers: { "content-type": "application/json" } }), + ) + }), ), ) - it.effect("generates Wan 2.7 through the international synchronous route", () => +describe("Alibaba Images", () => { + it.effect("supports open options and arbitrary models", () => Effect.gen(function* () { const result = yield* Image.generate({ - model: Alibaba.configure({ apiKey: "test", baseURL: "https://dashscope-intl.test/api/v1" }).image( - "wan2.7-image-pro", - ), - prompt: "A flower shop with a wooden door", - count: 1, - seed: 7, - providerOptions: { - alibaba: { - wan: { - resolution: "2K", - thinkingMode: true, - watermark: false, - colorPalette: [ - { hex: "#112233", ratio: "60.00%" }, - { hex: "#445566", ratio: "25.00%" }, - { hex: "#778899", ratio: "15.00%" }, - ], - }, + model: Alibaba.configure({ apiKey: "test" }).image("future-model-id"), + prompt: "A robot", + options: { + resolution: "2K", + negativePrompt: "blurry", + negative_prompt: "native wins", + future_parameter: true, + }, + http: { + body: { + model: "ignored-model", + input: { messages: [] }, + parameters: { seed: 7 }, }, }, }) - - expect(result.images).toHaveLength(1) - expect(result.image?.mediaType).toBe("image/png") - expect(result.usage?.totalTokens).toBe(12) - expect(result.usage?.providerMetadata).toEqual({ alibaba: response("wan").usage }) + expect(result.image?.expiresAt).toBe("2030-01-01T00:00:00.000Z") }).pipe( Effect.provide( - ImageClient.layer.pipe( - Layer.provide( - dynamicResponse((input) => - Effect.gen(function* () { - const request = yield* HttpClientRequest.toWeb(input.request).pipe(Effect.orDie) - expect(request.url).toBe( - "https://dashscope-intl.test/api/v1/services/aigc/multimodal-generation/generation", - ) - expect(JSON.parse(input.text)).toEqual({ - model: "wan2.7-image-pro", - input: { messages: [{ role: "user", content: [{ text: "A flower shop with a wooden door" }] }] }, - parameters: { - size: "2K", - n: 1, - thinking_mode: true, - color_palette: [ - { hex: "#112233", ratio: "60.00%" }, - { hex: "#445566", ratio: "25.00%" }, - { hex: "#778899", ratio: "15.00%" }, - ], - watermark: false, - seed: 7, - }, - }) - return input.respond(JSON.stringify(response("wan")), { - headers: { "content-type": "application/json" }, - }) - }), - ), - ), - ), + layer((body) => { + expect(body.model).toBe("future-model-id") + expect(body.input).toEqual({ messages: [{ role: "user", content: [{ text: "A robot" }] }] }) + expect(body.parameters).toEqual({ + size: "2K", + negative_prompt: "native wins", + future_parameter: true, + seed: 7, + }) + }), ), ), ) - it.effect("surfaces Alibaba error envelopes as typed provider errors", () => + it.effect("lowers ordered image inputs before text", () => Image.generate({ model: Alibaba.configure({ apiKey: "test" }).image("qwen-image-2.0"), - prompt: "A robot", + prompt: "Combine these", + images: [ + ImageInput.url("https://example.com/first.png"), + ImageInput.bytes(Uint8Array.from([1, 2, 3]), "image/png"), + ], }).pipe( - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error.reason._tag).toBe("UnknownProvider") - expect(error.message).toContain("InvalidParameter: invalid prompt") - }), - ), Effect.provide( - ImageClient.layer.pipe( - Layer.provide( - dynamicResponse((input) => - Effect.succeed( - input.respond( - JSON.stringify({ request_id: "request-error", code: "InvalidParameter", message: "invalid prompt" }), - { headers: { "content-type": "application/json" } }, - ), - ), - ), - ), - ), + layer((body) => { + const input = body.input as { messages: Array<{ content: unknown }> } + expect(input.messages[0].content).toEqual([ + { image: "https://example.com/first.png" }, + { image: "data:image/png;base64,AQID" }, + { text: "Combine these" }, + ]) + }), ), ), ) - it.effect("rejects parameters overlays owned by the protocol", () => + it.effect("rejects provider file inputs", () => Image.generate({ - model: Alibaba.configure({ apiKey: "test" }).image("qwen-image-2.0"), + model: Alibaba.configure({ apiKey: "test" }).image("any-model"), prompt: "A robot", - http: { body: { parameters: { watermark: true } } }, + images: [ImageInput.file("file_123")], }).pipe( Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error.reason._tag).toBe("InvalidRequest") - expect(error.message).toContain("http.body cannot overlay protocol-owned field(s): parameters") - }), - ), + Effect.tap((error) => Effect.sync(() => expect(error.reason._tag).toBe("InvalidRequest"))), Effect.provide( - ImageClient.layer.pipe( - Layer.provide( - dynamicResponse(() => Effect.die("reserved body validation must happen before the request is sent")), - ), - ), + ImageClient.layer.pipe(Layer.provide(dynamicResponse(() => Effect.die("must fail before network I/O")))), ), ), ) - it.effect("ignores finite expiration values outside the Date range", () => + it.effect("ignores expiration values outside the Date range", () => Effect.gen(function* () { const result = yield* Image.generate({ - model: Alibaba.configure({ apiKey: "test" }).image("qwen-image-2.0"), + model: Alibaba.configure({ apiKey: "test" }).image("any-model"), prompt: "A robot", }) - expect(result.image?.expiresAt).toBeUndefined() - expect(result.image?.providerMetadata?.alibaba?.expiresAt).toBeUndefined() - }).pipe( - Effect.provide( - ImageClient.layer.pipe( - Layer.provide( - dynamicResponse((input) => { - const payload = response("qwen") - payload.output.choices[0].message.content[0].image = - "https://dashscope-result-intl.oss-cn-singapore.aliyuncs.com/result.png?Expires=9007199254740991" - return Effect.succeed( - input.respond(JSON.stringify(payload), { headers: { "content-type": "application/json" } }), - ) - }), - ), - ), - ), - ), + expect(result.usage?.totalTokens).toBe(12) + }).pipe(Effect.provide(layer(() => {}, "https://example.com/result.png?Expires=9007199254740991"))), ) })