diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a37d97d..1f711a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.26.10" + go-version: "1.26.11" - name: Download dependencies run: go mod download diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5d9077d..1bd0561 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ and pull requests are welcome. ## Development -The project requires Go 1.26.10 or newer. +The project requires Go 1.26.11 or newer. ```sh go test ./... diff --git a/README.md b/README.md index 6e5eda7..f4675f4 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ print(response.choices[0].message.content) | --- | --- | | `GET /v1/models` | Lists the available models, prices and capabilities. | | `POST /v1/chat/completions` | Creates chat completions; supports streaming. | +| `POST /v1/images/generations` | Generates images with image models (OpenAI images API, inline `b64_json` only; H3 per token; `Idempotency-Key` supported). | Supported capabilities depend on the selected model and can include streaming, tool calls, parallel tool calls, structured outputs and reasoning. The current @@ -91,6 +92,16 @@ keep a valid `Retry-After` header. Errors include `origin` (`gateway` or details are not exposed. A configured fallback provider is tried only after a rate limit or a server-side failure, never after a rejected request. +Image generation follows the OpenAI images API: `model`, `prompt`, and `n` +(1 to 4). Images come back inline as `b64_json` and are not stored; a prompt +the provider refuses returns 422 `content_policy_violation` and costs +nothing. Send an `Idempotency-Key` header to retry safely: a retry of a +finished request returns the same images without a second charge. +`IMAGE_GENERATION_TIMEOUT_SECONDS` (default 660) bounds one generation and +`IMAGE_GENERATION_CONCURRENCY_PER_ACCOUNT` (default 2) how many one account +runs at once. Operator notes are in the Nexus repository's +`docs/image-generation.md`. + Responses have no limit on total duration. Two stall guards apply instead: `PROVIDER_SILENCE_TIMEOUT_SECONDS` (default 900) ends a request when the provider sends nothing, neither headers nor bytes, for that long, and @@ -150,3 +161,18 @@ provider enforces it. Metering holds credit for that limit, so a guest's hold fits its credit; other requests without `max_tokens` hold the model's maximum output. An invalid value stops the gateway at startup. The Nexus repository's `docs/guest-chat.md` describes guest chat. + +### Image billing + +H3 uses separate retail +rates for text input, reference-image input and image output tokens. Model discovery +publishes `image_input_price_per_1m_tokens_cents` alongside text-input/output rates, +and `image_max_cost_per_image_cents` for the maximum base charge and credit hold. +Exact token counts are validated from MiniMax usage and sent privately to metering. +Provider token costs remain independent of retail prices. + +Successful responses include `nexus_billing: {"charged_cents": 4.3, "currency": "EUR"}` +when metering returns a charge receipt. It is the amount actually deducted, including +any prepaid surcharge. Idempotent replays include the same receipt without charging +again. Failed/refused requests have no retail charge. Billing uses the rates and +ceiling saved when the hold was admitted; completion releases the unused hold. diff --git a/apps/gateway/Dockerfile b/apps/gateway/Dockerfile index a19b425..225730f 100644 --- a/apps/gateway/Dockerfile +++ b/apps/gateway/Dockerfile @@ -1,5 +1,5 @@ # ---- Build stage ---- -FROM golang:1.26.10-alpine AS builder +FROM golang:1.26.11-alpine AS builder WORKDIR /app diff --git a/apps/gateway/Dockerfile.enclave b/apps/gateway/Dockerfile.enclave index 5ecd18b..89710b0 100644 --- a/apps/gateway/Dockerfile.enclave +++ b/apps/gateway/Dockerfile.enclave @@ -7,7 +7,7 @@ # deploy/nitro/verify-build.sh. Build this image with `docker buildx build` # only -- a plain `docker build` skips the timestamp normalisation and silently # produces different measurements. -FROM golang:1.26.10-alpine3.24@sha256:27f2d3b83cfc5803c8078f8e730af0f7c431bf067afaec8f60a36fb763187cba AS builder +FROM golang:1.26.11-alpine3.24@sha256:ae52b5bc6ed045a9ed26ab03dc19327a802a9cc9aa8b27ed57f5c6b60579b0bb AS builder ARG SOURCE_REVISION ARG AWS_REGION diff --git a/apps/gateway/internal/adapters/http/dto/models_response.go b/apps/gateway/internal/adapters/http/dto/models_response.go index af2ff09..eb56c38 100644 --- a/apps/gateway/internal/adapters/http/dto/models_response.go +++ b/apps/gateway/internal/adapters/http/dto/models_response.go @@ -29,6 +29,11 @@ type ModelData struct { OutputPricePer1MTokensUSD *float64 `json:"output_price_per_1m_tokens_usd,omitempty"` CacheReadPricePer1MTokensUSD *float64 `json:"cache_read_price_per_1m_tokens_usd,omitempty"` CacheWritePricePer1MTokensUSD *float64 `json:"cache_write_price_per_1m_tokens_usd,omitempty"` + // Image models use three token rates plus a + // maximum per image. The maximum excludes any account surcharge. + ImageInputPricePer1MTokensCents *int64 `json:"image_input_price_per_1m_tokens_cents,omitempty"` + ImageInputPricePer1MTokensUSD *float64 `json:"image_input_price_per_1m_tokens_usd,omitempty"` + ImageMaxCostPerImageCents *float64 `json:"image_max_cost_per_image_cents,omitempty"` // Capabilities is what the operator states the model accepts, using // models.dev field names. A missing field means not stated. Capabilities json.RawMessage `json:"capabilities,omitempty"` diff --git a/apps/gateway/internal/adapters/http/handlers/image_generations_handler.go b/apps/gateway/internal/adapters/http/handlers/image_generations_handler.go new file mode 100644 index 0000000..86685c3 --- /dev/null +++ b/apps/gateway/internal/adapters/http/handlers/image_generations_handler.go @@ -0,0 +1,58 @@ +package handlers + +import ( + "io" + "net/http" + "strings" + + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/adapters/http/mapper" + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/application/ports" + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/application/services" + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" +) + +// ImageGenerationsHandler handles POST /v1/images/generations, the OpenAI +// images API. Images come back inline (b64_json) and are not stored. +type ImageGenerationsHandler struct { + service *services.GenerateService + logger ports.Logger +} + +func NewImageGenerationsHandler(service *services.GenerateService, logger ports.Logger) *ImageGenerationsHandler { + return &ImageGenerationsHandler{service: service, logger: logger} +} + +func (h *ImageGenerationsHandler) Handle(w http.ResponseWriter, r *http.Request) { + token, err := ExtractBearerToken(r) + if err != nil { + rejectRequest(w, r, h.logger, domain.EndpointImageGenerations, err) + return + } + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 32<<20)) + if err != nil { + rejectRequest(w, r, h.logger, domain.EndpointImageGenerations, domain.ErrInvalidField("failed to read request body")) + return + } + req, err := mapper.ImageGenerationRequestToDomain(body) + if err != nil { + rejectRequest(w, r, h.logger, domain.EndpointImageGenerations, err) + return + } + req.ImageEdit = r.URL.Path == "/v1/images/edits" + if req.ImageEdit != mapper.HasImageInputs(body) { + rejectRequest(w, r, h.logger, domain.EndpointImageGenerations, domain.ErrInvalidField("send reference images to /v1/images/edits; generations takes text only")) + return + } + call, err := h.service.GenerateImages(r.Context(), body, req, token, strings.TrimSpace(r.Header.Get("Idempotency-Key"))) + if err != nil { + WriteErrorWithLog(w, r, h.logger, err) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + if call.Replayed { + w.Header().Set("Idempotent-Replayed", "true") + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write(call.Body) +} diff --git a/apps/gateway/internal/adapters/http/handlers/models_handler.go b/apps/gateway/internal/adapters/http/handlers/models_handler.go index 5b3a533..ce1ff6d 100644 --- a/apps/gateway/internal/adapters/http/handlers/models_handler.go +++ b/apps/gateway/internal/adapters/http/handlers/models_handler.go @@ -53,7 +53,7 @@ func toPublicModelData(m domain.PublicModel, eurToUSDRate float64) dto.ModelData inputPrice := eurToCents(m.InputPricePerMillion) outputPrice := eurToCents(m.OutputPricePerMillion) proofMode := m.EffectiveProofMode() - return dto.ModelData{ + data := dto.ModelData{ ID: m.PublicModelID, Object: "model", Kind: string(domain.CatalogKindPublicModel), @@ -73,11 +73,21 @@ func toPublicModelData(m domain.PublicModel, eurToUSDRate float64) dto.ModelData OutputPricePer1MTokensUSD: priceToUSDPtr(m.Currency, m.OutputPricePerMillion, eurToUSDRate), CacheReadPricePer1MTokensUSD: pricePtrToUSDPtr(m.Currency, m.CacheReadPricePerMillion, eurToUSDRate), CacheWritePricePer1MTokensUSD: pricePtrToUSDPtr(m.Currency, m.CacheWritePricePerMillion, eurToUSDRate), + ImageInputPricePer1MTokensCents: eurPtrToCentsPtr(m.ImageInputPricePerMillion), + ImageInputPricePer1MTokensUSD: pricePtrToUSDPtr(m.Currency, m.ImageInputPricePerMillion, eurToUSDRate), + ImageMaxCostPerImageCents: fractionalCents(m.ImageMaxCostPerImage), Capabilities: publishedCapabilities(m), Endpoints: modelEndpoints(m), ProofMode: proofMode, ProofsEnabled: domain.ProofModeEnabled(proofMode), } + if !m.SupportsChatCompletions && m.SupportsEndpoint(domain.EndpointImageGenerations) { + // Image requests have no chat token limits; token-priced image models + // still publish their retail text/reference/output rates. + data.ContextSize, data.MaxOutputTokens = nil, nil + + } + return data } func toRouterData(r domain.RouterEntry) dto.ModelData { @@ -112,6 +122,12 @@ func modelEndpoints(m domain.PublicModel) []string { if m.SupportsChatCompletions { endpoints = append(endpoints, "chat/completions") } + if m.SupportsEndpoint(domain.EndpointImageGenerations) { + endpoints = append(endpoints, "images/generations") + if m.SupportsImageEdits() { + endpoints = append(endpoints, "images/edits") + } + } return endpoints } @@ -119,6 +135,16 @@ func eurToCents(price decimal.Decimal) int64 { return price.Mul(decimal.NewFromInt(100)).Round(0).IntPart() } +// fractionalCents is a price in cents to microcent precision: per-image prices are +// often fractions of a cent. +func fractionalCents(price *decimal.Decimal) *float64 { + if price == nil { + return nil + } + cents, _ := price.Mul(decimal.NewFromInt(100)).Round(6).Float64() + return ¢s +} + func eurPtrToCentsPtr(price *decimal.Decimal) *int64 { if price == nil { return nil @@ -145,7 +171,7 @@ func priceToUSDPtr(currency string, price decimal.Decimal, eurToUSDRate float64) if factor <= 0 { return nil } - usd, _ := price.Mul(decimal.NewFromFloat(factor)).Round(4).Float64() + usd, _ := price.Mul(decimal.NewFromFloat(factor)).Round(8).Float64() return &usd } diff --git a/apps/gateway/internal/adapters/http/handlers/models_handler_test.go b/apps/gateway/internal/adapters/http/handlers/models_handler_test.go index b0ceae7..74d1b44 100644 --- a/apps/gateway/internal/adapters/http/handlers/models_handler_test.go +++ b/apps/gateway/internal/adapters/http/handlers/models_handler_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" + "github.com/shopspring/decimal" ) // The operator's record is published unchanged, including fields this Gateway @@ -26,3 +27,26 @@ func TestModelsHandler_PublishesCapabilitiesUnchanged(t *testing.T) { t.Fatalf("published capabilities nobody stated: %s", body) } } + +func TestToPublicModelData_ImageModel(t *testing.T) { + rate := decimal.RequireFromString("1.5") + cap := decimal.RequireFromString("0.50360083") + data := toPublicModelData(domain.PublicModel{ + PublicModelID: "minimax/h3-image", Currency: "EUR", UpstreamModelName: "MiniMax-H3-Image", + ProviderConfig: domain.ProviderConfig{ProviderName: "minimax"}, + SupportsImageGenerations: true, ImageInputPricePerMillion: &rate, ImageMaxCostPerImage: &cap, + InputPricePerMillion: decimal.NewFromInt(2), OutputPricePerMillion: decimal.NewFromInt(4), + }, 1.1) + if len(data.Endpoints) != 2 || data.Endpoints[0] != "images/generations" || data.Endpoints[1] != "images/edits" { + t.Fatalf("endpoints = %v", data.Endpoints) + } + if data.ImageInputPricePer1MTokensCents == nil || *data.ImageInputPricePer1MTokensCents != 150 { + t.Fatalf("reference token rate = %v", data.ImageInputPricePer1MTokensCents) + } + if data.ImageMaxCostPerImageCents == nil || *data.ImageMaxCostPerImageCents != 50.360083 { + t.Fatalf("ceiling precision = %v", data.ImageMaxCostPerImageCents) + } + if data.ContextSize != nil || data.InputPricePer1MTokensCents == nil || *data.InputPricePer1MTokensCents != 200 { + t.Fatal("image model needs token prices and no context size") + } +} diff --git a/apps/gateway/internal/adapters/http/mapper/image_to_domain.go b/apps/gateway/internal/adapters/http/mapper/image_to_domain.go new file mode 100644 index 0000000..ba00d98 --- /dev/null +++ b/apps/gateway/internal/adapters/http/mapper/image_to_domain.go @@ -0,0 +1,116 @@ +package mapper + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + _ "golang.org/x/image/webp" + "image" + _ "image/jpeg" + _ "image/png" + "strings" + + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" +) + +// MaxImagesPerRequest bounds n on /v1/images/generations, so one request's +// hold and response stay small. +const MaxImagesPerRequest = 4 + +// imageRequest is what the gateway reads from an images/generations body. +// The body itself is forwarded as sent. +type imageRequest struct { + Model string `json:"model"` + Prompt *string `json:"prompt"` + N json.RawMessage `json:"n"` + Stream *bool `json:"stream"` + ResponseFormat *string `json:"response_format"` +} + +// ImageGenerationRequestToDomain validates a /v1/images/generations body: +// a model, a prompt, and how many images (n, 1 by default). Images always +// come back inline as b64_json; streaming partial images is not offered. +func ImageGenerationRequestToDomain(raw json.RawMessage) (domain.GenerateRequest, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return domain.GenerateRequest{}, domain.ErrInvalidField("invalid JSON body") + } + var req imageRequest + if err := json.Unmarshal(raw, &req); err != nil { + return domain.GenerateRequest{}, domain.ErrInvalidField("invalid request body: " + err.Error()) + } + if strings.TrimSpace(req.Model) == "" { + return domain.GenerateRequest{}, domain.ErrInvalidField("model is required") + } + if req.Prompt == nil || strings.TrimSpace(*req.Prompt) == "" { + return domain.GenerateRequest{}, domain.ErrInvalidField("prompt is required") + } + if req.Stream != nil && *req.Stream { + return domain.GenerateRequest{}, domain.ErrUnsupportedFeature("streaming image generation") + } + if _, ok := fields["partial_images"]; ok { + return domain.GenerateRequest{}, domain.ErrUnsupportedFeature("partial_images") + } + if req.ResponseFormat != nil && *req.ResponseFormat != "b64_json" { + return domain.GenerateRequest{}, domain.ErrInvalidField("response_format must be b64_json: Nexus returns images inline and stores none") + } + if err := validateImageInputs(fields); err != nil { + return domain.GenerateRequest{}, err + } + n := 1 + if len(req.N) > 0 && string(req.N) != "null" { + if err := json.Unmarshal(req.N, &n); err != nil || n < 1 || n > MaxImagesPerRequest { + return domain.GenerateRequest{}, domain.ErrInvalidField(fmt.Sprintf("n must be a whole number from 1 to %d", MaxImagesPerRequest)) + } + } + return domain.GenerateRequest{ + PublicModelID: req.Model, + RequestedModelID: req.Model, + Images: &n, + }, nil +} + +// HasImageInputs distinguishes editing from text-to-image requests. +func HasImageInputs(raw []byte) bool { + var body map[string]json.RawMessage + _ = json.Unmarshal(raw, &body) + _, ok := body["images"] + return ok +} + +func validateImageInputs(fields map[string]json.RawMessage) error { + raw, exists := fields["images"] + if !exists { + return nil + } + var images []struct { + ImageURL string `json:"image_url"` + } + if json.Unmarshal(raw, &images) != nil || len(images) < 1 || len(images) > 4 { + return domain.ErrInvalidField("images must contain 1 to 4 inline reference images") + } + total := 0 + for _, item := range images { + header, encoded, ok := strings.Cut(item.ImageURL, ",") + if !ok || (header != "data:image/png;base64" && header != "data:image/jpeg;base64" && header != "data:image/webp;base64") { + return domain.ErrInvalidField("reference images must be PNG, JPEG or WebP base64 data URLs; remote URLs are not accepted") + } + if len(encoded) > 14<<20 { + return domain.ErrInvalidField("each reference image must be at most 10 MiB") + } + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil || len(decoded) > 10<<20 { + return domain.ErrInvalidField("invalid reference image or image exceeds 10 MiB") + } + total += len(decoded) + cfg, format, err := image.DecodeConfig(bytes.NewReader(decoded)) + if err != nil || "data:image/"+format+";base64" != header || cfg.Width < 128 || cfg.Height < 128 || cfg.Width > 8192 || cfg.Height > 8192 || cfg.Width > 4*cfg.Height || cfg.Height > 4*cfg.Width { + return domain.ErrInvalidField("reference images must be 128–8192 px on each side, with an aspect ratio at most 4:1") + } + } + if total > 20<<20 { + return domain.ErrInvalidField("reference images must total at most 20 MiB") + } + return nil +} diff --git a/apps/gateway/internal/adapters/http/mapper/image_to_domain_test.go b/apps/gateway/internal/adapters/http/mapper/image_to_domain_test.go new file mode 100644 index 0000000..1a01c9b --- /dev/null +++ b/apps/gateway/internal/adapters/http/mapper/image_to_domain_test.go @@ -0,0 +1,73 @@ +package mapper + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "image" + "image/png" + "testing" +) + +func TestImageGenerationRequestToDomain(t *testing.T) { + req, err := ImageGenerationRequestToDomain([]byte(`{"model":"flux","prompt":"a fox"}`)) + if err != nil || req.PublicModelID != "flux" || req.Images == nil || *req.Images != 1 { + t.Fatalf("got %+v, %v; want model flux and 1 image by default", req, err) + } + req, err = ImageGenerationRequestToDomain([]byte(`{"model":"flux","prompt":"a fox","n":3,"size":"1024x1024","response_format":"b64_json"}`)) + if err != nil || *req.Images != 3 { + t.Fatalf("got %+v, %v; want 3 images", req, err) + } + for name, body := range map[string]string{ + "no model": `{"prompt":"a fox"}`, + "no prompt": `{"model":"flux"}`, + "blank prompt": `{"model":"flux","prompt":" "}`, + "n zero": `{"model":"flux","prompt":"x","n":0}`, + "n too large": `{"model":"flux","prompt":"x","n":5}`, + "n fractional": `{"model":"flux","prompt":"x","n":1.5}`, + "url output": `{"model":"flux","prompt":"x","response_format":"url"}`, + "streaming": `{"model":"flux","prompt":"x","stream":true}`, + "partial images": `{"model":"flux","prompt":"x","partial_images":2}`, + "invalid json": `{`, + "prompt not text": `{"model":"flux","prompt":7}`, + } { + if _, err := ImageGenerationRequestToDomain([]byte(body)); err == nil { + t.Errorf("%s: want an error", name) + } + } +} + +func TestImageEditRejectsRemoteAndInvalidReferences(t *testing.T) { + for _, images := range []string{`[]`, `null`, `[{"image_url":"https://example.com/image.png"}]`, `[{"image_url":"data:image/png;base64,not-an-image"}]`} { + if _, err := ImageGenerationRequestToDomain([]byte(`{"model":"minimax/h3-image","prompt":"edit","images":` + images + `}`)); err == nil { + t.Errorf("accepted %s", images) + } + } +} + +func TestImageEditValidatesReferenceDimensionsAndMIME(t *testing.T) { + for _, tc := range []struct { + name string + width, height int + mime string + valid bool + }{ + {"valid", 128, 128, "png", true}, + {"too small", 127, 128, "png", false}, + {"too wide", 640, 128, "png", false}, + {"MIME mismatch", 128, 128, "jpeg", false}, + } { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + if err := png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, tc.width, tc.height))); err != nil { + t.Fatal(err) + } + dataURL := "data:image/" + tc.mime + ";base64," + base64.StdEncoding.EncodeToString(buf.Bytes()) + body, _ := json.Marshal(map[string]any{"model": "minimax/h3-image", "prompt": "edit", "images": []map[string]string{{"image_url": dataURL}}}) + _, err := ImageGenerationRequestToDomain(body) + if (err == nil) != tc.valid { + t.Fatalf("valid=%v, error=%v", tc.valid, err) + } + }) + } +} diff --git a/apps/gateway/internal/adapters/http/middleware/cors.go b/apps/gateway/internal/adapters/http/middleware/cors.go index 04fd8c6..100340c 100644 --- a/apps/gateway/internal/adapters/http/middleware/cors.go +++ b/apps/gateway/internal/adapters/http/middleware/cors.go @@ -5,7 +5,8 @@ import "net/http" // CORS adds permissive CORS headers so browser SPAs can call the gateway // directly — both the public landing page hitting the unauthenticated // GET /v1/models endpoint and the user-ui chat playground hitting -// /v1/chat/completions with a Bearer API key. Origin is reflected only when +// /v1/chat/completions and /v1/images/generations (with an Idempotency-Key) +// with a Bearer API key. Origin is reflected only when // present, with `Vary: Origin` to keep intermediate caches correct. func CORS(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -15,8 +16,8 @@ func CORS(next http.Handler) http.Handler { w.Header().Set("Vary", "Origin") } w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Ehbp-Encapsulated-Key, X-Request-ID") - w.Header().Set("Access-Control-Expose-Headers", "Ehbp-Response-Nonce, X-Request-ID") + w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Ehbp-Encapsulated-Key, Idempotency-Key, X-Request-ID") + w.Header().Set("Access-Control-Expose-Headers", "Ehbp-Response-Nonce, Idempotent-Replayed, Retry-After, X-Request-ID") w.Header().Set("Access-Control-Max-Age", "86400") if r.Method == http.MethodOptions { diff --git a/apps/gateway/internal/adapters/http/middleware/cors_test.go b/apps/gateway/internal/adapters/http/middleware/cors_test.go new file mode 100644 index 0000000..4cb7328 --- /dev/null +++ b/apps/gateway/internal/adapters/http/middleware/cors_test.go @@ -0,0 +1,25 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// The web chat sends an Idempotency-Key with image requests; a browser only +// sends it when the preflight allows it. +func TestCORSAllowsIdempotencyKey(t *testing.T) { + handler := CORS(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + req := httptest.NewRequest(http.MethodOptions, "/v1/images/generations", nil) + req.Header.Set("Origin", "https://nexus.dappnode.com") + req.Header.Set("Access-Control-Request-Headers", "authorization, content-type, idempotency-key") + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + if !strings.Contains(rec.Header().Get("Access-Control-Allow-Headers"), "Idempotency-Key") { + t.Fatalf("Allow-Headers = %q, want Idempotency-Key", rec.Header().Get("Access-Control-Allow-Headers")) + } + if !strings.Contains(rec.Header().Get("Access-Control-Expose-Headers"), "Idempotent-Replayed") { + t.Fatal("the replay header must be readable by the browser") + } +} diff --git a/apps/gateway/internal/adapters/http/router.go b/apps/gateway/internal/adapters/http/router.go index 701fee9..fa9f7c0 100644 --- a/apps/gateway/internal/adapters/http/router.go +++ b/apps/gateway/internal/adapters/http/router.go @@ -13,6 +13,7 @@ func NewRouter( health *handlers.HealthHandler, models *handlers.ModelsHandler, chatCompletions *handlers.ChatCompletionsHandler, + imageGenerations *handlers.ImageGenerationsHandler, systemOne *handlers.SystemOneHandler, confidentialChat *handlers.ConfidentialChatCompletionsHandler, tinfoil *handlers.TinfoilHandler, @@ -24,6 +25,10 @@ func NewRouter( mux.HandleFunc("GET /healthz", health.Handle) mux.HandleFunc("GET /v1/models", models.Handle) mux.HandleFunc("POST /v1/chat/completions", chatCompletions.Handle) + if imageGenerations != nil { + mux.HandleFunc("POST /v1/images/generations", imageGenerations.Handle) + mux.HandleFunc("POST /v1/images/edits", imageGenerations.Handle) + } if systemOne != nil { mux.HandleFunc("POST /v1/systemone", systemOne.Handle) } diff --git a/apps/gateway/internal/adapters/http/router_test.go b/apps/gateway/internal/adapters/http/router_test.go index 9c95f10..5ce552c 100644 --- a/apps/gateway/internal/adapters/http/router_test.go +++ b/apps/gateway/internal/adapters/http/router_test.go @@ -24,6 +24,7 @@ func TestNewRouter_PreservesLegacyChatRouteWithoutConfidentialHandler(t *testing handlers.NewHealthHandler(), handlers.NewModelsHandler(nil), handlers.NewChatCompletionsHandler(nil, logger), + nil, handlers.NewSystemOneHandler(nil, logger), nil, nil, @@ -63,7 +64,7 @@ func TestNewRouter_PreservesLegacyChatRouteWithoutConfidentialHandler(t *testing // in between hands the mux a copy of the request. func TestNewRouter_LabelsMetricsWithTheMatchedRoute(t *testing.T) { logger := &routerTestLogger{} - router := NewRouter(handlers.NewHealthHandler(), handlers.NewModelsHandler(nil), handlers.NewChatCompletionsHandler(nil, logger), nil, nil, nil, nil, logger) + router := NewRouter(handlers.NewHealthHandler(), handlers.NewModelsHandler(nil), handlers.NewChatCompletionsHandler(nil, logger), nil, nil, nil, nil, nil, logger) health := metrics.HTTPRequestsTotal.WithLabelValues("GET /healthz", http.MethodGet, "2xx") unmatched := metrics.HTTPRequestsTotal.WithLabelValues("unmatched", http.MethodGet, "4xx") healthBefore, unmatchedBefore := testutil.ToFloat64(health), testutil.ToFloat64(unmatched) diff --git a/apps/gateway/internal/adapters/metering/client.go b/apps/gateway/internal/adapters/metering/client.go index 6407457..ec19f84 100644 --- a/apps/gateway/internal/adapters/metering/client.go +++ b/apps/gateway/internal/adapters/metering/client.go @@ -59,6 +59,7 @@ func (c *Client) Reserve(ctx context.Context, auth domain.AuthContext, endpoint Stream: req.Stream, MaxOutputTokens: req.MaxOutputTokens, EstimatedInputTokens: req.EstimatedInputTokens, + Images: req.Images, } var res reserveResponse sent := time.Now() // A lease measured from here never outlasts Metering's. @@ -75,6 +76,11 @@ func (c *Client) Reserve(ctx context.Context, auth domain.AuthContext, endpoint } func (c *Client) RecordSuccess(ctx context.Context, reservationID string, _ domain.AuthContext, _ string, _ domain.GenerateRequest, resp domain.GenerateResult, model domain.PublicModel, latencyMs int64) error { + _, err := c.CompleteWithReceipt(ctx, reservationID, resp, model, latencyMs) + return err +} + +func (c *Client) CompleteWithReceipt(ctx context.Context, reservationID string, resp domain.GenerateResult, model domain.PublicModel, latencyMs int64) (*int64, error) { providerName := resp.ProviderName if providerName == "" { providerName = model.ProviderConfig.ProviderName @@ -92,7 +98,8 @@ func (c *Client) RecordSuccess(ctx context.Context, reservationID string, _ doma LatencyMs: latencyMs, } var res completeResponse - return c.postJSON(ctx, "/internal/metering/reservations/"+reservationID+"/complete", payload, &res, 2) + err := c.postJSON(ctx, "/internal/metering/reservations/"+reservationID+"/complete", payload, &res, 2) + return res.ChargedMicrocents, err } func (c *Client) RecordFailure(ctx context.Context, reservationID *string, auth *domain.AuthContext, endpoint string, req *domain.GenerateRequest, model *domain.PublicModel, err error, partialUsage *domain.Usage, latencyMs int64) error { @@ -422,6 +429,7 @@ type reserveRequest struct { Stream bool `json:"stream"` MaxOutputTokens *int `json:"max_output_tokens,omitempty"` EstimatedInputTokens *int64 `json:"estimated_input_tokens,omitempty"` + Images *int `json:"images,omitempty"` } type reserveResponse struct { @@ -439,7 +447,8 @@ type completeRequest struct { } type completeResponse struct { - UsageEventID string `json:"usage_event_id,omitempty"` + ChargedMicrocents *int64 `json:"charged_microcents,omitempty"` + UsageEventID string `json:"usage_event_id,omitempty"` } type failureRequest struct { diff --git a/apps/gateway/internal/adapters/metering/runtime_client.go b/apps/gateway/internal/adapters/metering/runtime_client.go index ed7f7d4..8b80fea 100644 --- a/apps/gateway/internal/adapters/metering/runtime_client.go +++ b/apps/gateway/internal/adapters/metering/runtime_client.go @@ -159,6 +159,12 @@ func validateAuthenticateResponse(response authenticateResponse) *domain.Gateway return nil } +// imageOnly is an image model that serves no chat: it has no context window +// or output limit. +func imageOnly(model runtimePublicModel) bool { + return model.SupportsImageGenerations && model.SupportsChatCompletions != nil && !*model.SupportsChatCompletions +} + func validateRuntimePublicModel(model runtimePublicModel, expectedPublicModelID string) *domain.GatewayError { switch { case strings.TrimSpace(model.ID) == "": @@ -202,10 +208,16 @@ func validateRuntimePublicModel(model runtimePublicModel, expectedPublicModelID return runtimeContractError("public model", "supports_reasoning is required") case model.ProofMode != domain.ProofModeNone && model.ProofMode != domain.ProofModeTinfoilAttestedTransport: return runtimeContractError("public model", "proof_mode is invalid") - case model.MaxContextWindow <= 0: + case model.MaxContextWindow <= 0 && !imageOnly(model): return runtimeContractError("public model", "max_context_window must be positive") - case model.MaxOutputTokens <= 0: + case model.MaxOutputTokens <= 0 && !imageOnly(model): return runtimeContractError("public model", "max_output_tokens must be positive") + case model.MaxContextWindow < 0 || model.MaxOutputTokens < 0: + return runtimeContractError("public model", "token limits must not be negative") + case model.SupportsImageGenerations && (model.ImageInputPricePer1MTokensMicrocents == nil || model.ImageMaxCostPerImageMicrocents == nil): + return runtimeContractError("public model", "image models require reference token pricing and a spending ceiling") + case model.ImageInputPricePer1MTokensMicrocents != nil && (*model.ImageInputPricePer1MTokensMicrocents < 0 || model.ImageMaxCostPerImageMicrocents == nil || *model.ImageMaxCostPerImageMicrocents <= 0): + return runtimeContractError("public model", "image token prices require a non-negative reference rate and a positive spending ceiling") case model.InputPricePer1MTokensMicrocents == nil: return runtimeContractError("public model", "input price is required") case *model.InputPricePer1MTokensMicrocents < 0: @@ -379,6 +391,15 @@ func (m runtimePublicModel) toDomain() domain.PublicModel { price := priceFromMicrocents(*m.CacheWritePricePer1MTokensMicrocents) model.CacheWritePricePerMillion = &price } + if m.ImageInputPricePer1MTokensMicrocents != nil { + v := priceFromMicrocents(*m.ImageInputPricePer1MTokensMicrocents) + model.ImageInputPricePerMillion = &v + } + if m.ImageMaxCostPerImageMicrocents != nil { + v := priceFromMicrocents(*m.ImageMaxCostPerImageMicrocents) + model.ImageMaxCostPerImage = &v + } + model.SupportsImageGenerations = m.SupportsImageGenerations return model } @@ -429,32 +450,36 @@ type runtimeAPIKey struct { } type runtimePublicModel struct { - ID string `json:"id"` - PublicModelID string `json:"public_model_id"` - DisplayName string `json:"display_name"` - Description *string `json:"description,omitempty"` - ProviderModelID string `json:"provider_model_id"` - UpstreamModelName string `json:"upstream_model_name"` - ServiceTier *string `json:"service_tier,omitempty"` - ProviderConfig runtimeProviderConfig `json:"provider_config"` - Fallback *runtimeProviderTarget `json:"fallback,omitempty"` - SupportsChatCompletions *bool `json:"supports_chat_completions"` - SupportsChatCompletionsStream *bool `json:"supports_chat_completions_stream"` - SupportsTools *bool `json:"supports_tools"` - SupportsParallelToolCalls *bool `json:"supports_parallel_tool_calls"` - SupportsStructuredOutput *bool `json:"supports_structured_output"` - SupportsReasoning *bool `json:"supports_reasoning"` - Capabilities json.RawMessage `json:"capabilities,omitempty"` - ProofMode string `json:"proof_mode"` - MaxContextWindow int `json:"max_context_window"` - MaxOutputTokens int `json:"max_output_tokens"` - InputPricePer1MTokensMicrocents *int64 `json:"input_price_per_1m_tokens_microcents"` - OutputPricePer1MTokensMicrocents *int64 `json:"output_price_per_1m_tokens_microcents"` - CacheReadPricePer1MTokensMicrocents *int64 `json:"cache_read_price_per_1m_tokens_microcents,omitempty"` - CacheWritePricePer1MTokensMicrocents *int64 `json:"cache_write_price_per_1m_tokens_microcents,omitempty"` - Currency string `json:"currency"` - Active bool `json:"active"` - GuestAllowed bool `json:"guest_allowed"` + ID string `json:"id"` + PublicModelID string `json:"public_model_id"` + DisplayName string `json:"display_name"` + Description *string `json:"description,omitempty"` + ProviderModelID string `json:"provider_model_id"` + UpstreamModelName string `json:"upstream_model_name"` + ServiceTier *string `json:"service_tier,omitempty"` + ProviderConfig runtimeProviderConfig `json:"provider_config"` + Fallback *runtimeProviderTarget `json:"fallback,omitempty"` + SupportsChatCompletions *bool `json:"supports_chat_completions"` + SupportsChatCompletionsStream *bool `json:"supports_chat_completions_stream"` + SupportsTools *bool `json:"supports_tools"` + SupportsParallelToolCalls *bool `json:"supports_parallel_tool_calls"` + SupportsStructuredOutput *bool `json:"supports_structured_output"` + SupportsReasoning *bool `json:"supports_reasoning"` + // Absent from an older Metering, which has no image models. + SupportsImageGenerations bool `json:"supports_image_generations"` + Capabilities json.RawMessage `json:"capabilities,omitempty"` + ProofMode string `json:"proof_mode"` + MaxContextWindow int `json:"max_context_window"` + MaxOutputTokens int `json:"max_output_tokens"` + InputPricePer1MTokensMicrocents *int64 `json:"input_price_per_1m_tokens_microcents"` + OutputPricePer1MTokensMicrocents *int64 `json:"output_price_per_1m_tokens_microcents"` + CacheReadPricePer1MTokensMicrocents *int64 `json:"cache_read_price_per_1m_tokens_microcents,omitempty"` + CacheWritePricePer1MTokensMicrocents *int64 `json:"cache_write_price_per_1m_tokens_microcents,omitempty"` + ImageInputPricePer1MTokensMicrocents *int64 `json:"image_input_price_per_1m_tokens_microcents,omitempty"` + ImageMaxCostPerImageMicrocents *int64 `json:"image_max_cost_per_image_microcents,omitempty"` + Currency string `json:"currency"` + Active bool `json:"active"` + GuestAllowed bool `json:"guest_allowed"` } type runtimeProviderTarget struct { diff --git a/apps/gateway/internal/adapters/metering/runtime_client_image_test.go b/apps/gateway/internal/adapters/metering/runtime_client_image_test.go new file mode 100644 index 0000000..ff554fe --- /dev/null +++ b/apps/gateway/internal/adapters/metering/runtime_client_image_test.go @@ -0,0 +1,44 @@ +package metering + +import "testing" + +// An image-only model has no context window or output limit; one such model +// must not make the whole catalog invalid. +func TestValidateRuntimePublicModelAcceptsImageOnlyModel(t *testing.T) { + no, yes := false, true + zero, price := int64(0), int64(2_000_000) + model := runtimePublicModel{ + ID: "id", PublicModelID: "minimax/h3-image", DisplayName: "MiniMax H3 Image", ProviderModelID: "pm", UpstreamModelName: "MiniMax-H3-Image", + ProviderConfig: runtimeProviderConfig{ID: "pc", ProviderName: "minimax", BaseURL: "https://x", APIKeySecretRef: "MINIMAX_API_KEY", Active: true}, + Active: true, + SupportsChatCompletions: &no, + SupportsChatCompletionsStream: &no, SupportsTools: &no, SupportsParallelToolCalls: &no, + SupportsStructuredOutput: &no, SupportsReasoning: &no, + SupportsImageGenerations: true, ImageInputPricePer1MTokensMicrocents: &price, ImageMaxCostPerImageMicrocents: &price, + ProofMode: "none", Currency: "EUR", + InputPricePer1MTokensMicrocents: &zero, OutputPricePer1MTokensMicrocents: &zero, + } + if err := validateRuntimePublicModel(model, ""); err != nil { + t.Fatalf("image-only model rejected: %v", err) + } + for _, field := range []string{"reference rate", "ceiling"} { + invalid := model + if field == "reference rate" { + invalid.ImageInputPricePer1MTokensMicrocents = nil + } else { + invalid.ImageMaxCostPerImageMicrocents = nil + } + if err := validateRuntimePublicModel(invalid, ""); err == nil { + t.Fatalf("image model missing %s accepted", field) + } + } + model.SupportsChatCompletions = &yes + if err := validateRuntimePublicModel(model, ""); err == nil { + t.Fatal("a chat model still needs a context window") + } + negative := int64(-1) + model.SupportsChatCompletions, model.ImageInputPricePer1MTokensMicrocents = &no, &negative + if err := validateRuntimePublicModel(model, ""); err == nil { + t.Fatal("a negative image price must be rejected") + } +} diff --git a/apps/gateway/internal/adapters/observability/metrics/prometheus.go b/apps/gateway/internal/adapters/observability/metrics/prometheus.go index 370a4f8..2fcb42b 100644 --- a/apps/gateway/internal/adapters/observability/metrics/prometheus.go +++ b/apps/gateway/internal/adapters/observability/metrics/prometheus.go @@ -99,6 +99,14 @@ var ( }, []string{"direction", "model", "provider"}, ) + + ImagesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "nexus_gateway_images_total", + Help: "Total images generated, by model and provider.", + }, + []string{"model", "provider"}, + ) ) // Downstream dependency metrics. @@ -263,6 +271,7 @@ func init() { UpstreamLatency, TokensTotal, CacheTokensTotal, + ImagesTotal, MeteringRequests, MeteringDuration, RouterRequests, @@ -304,4 +313,7 @@ func RecordUsage(u *domain.Usage, model, provider string) { if v := u.CacheCreationTokens; v > 0 { CacheTokensTotal.WithLabelValues("write", model, provider).Add(float64(v)) } + if v := u.Images; v > 0 { + ImagesTotal.WithLabelValues(model, provider).Add(float64(v)) + } } diff --git a/apps/gateway/internal/adapters/providers/openai/adapter.go b/apps/gateway/internal/adapters/providers/openai/adapter.go index ac1f072..a1f6d31 100644 --- a/apps/gateway/internal/adapters/providers/openai/adapter.go +++ b/apps/gateway/internal/adapters/providers/openai/adapter.go @@ -16,8 +16,9 @@ import ( // Adapter is the OpenAI-compatible provider adapter. type Adapter struct { - client *Client - logger ports.Logger + client *Client + imageClient *Client + logger ports.Logger } const ( @@ -34,7 +35,7 @@ func NewAdapter(timeout time.Duration, logger ...ports.Logger) *Adapter { if len(logger) > 0 { l = logger[0] } - return &Adapter{client: NewClient(timeout), logger: l} + return &Adapter{client: NewClient(timeout), imageClient: NewClient(660 * time.Second), logger: l} } func missingProviderCredentialError(providerName string) *domain.GatewayError { diff --git a/apps/gateway/internal/adapters/providers/openai/client.go b/apps/gateway/internal/adapters/providers/openai/client.go index 0d5761a..ece0c3e 100644 --- a/apps/gateway/internal/adapters/providers/openai/client.go +++ b/apps/gateway/internal/adapters/providers/openai/client.go @@ -50,8 +50,17 @@ func (c *Client) Do(ctx context.Context, baseURL, apiKey string, body map[string return c.DoEndpoint(ctx, baseURL, "/chat/completions", apiKey, body) } +// maxJSONResponseSize bounds a non-streaming JSON response. +const maxJSONResponseSize = 10 * 1024 * 1024 + // DoEndpoint uses the same bounded, enclave-aware transport for JSON APIs. func (c *Client) DoEndpoint(ctx context.Context, baseURL, endpoint, apiKey string, body map[string]any) ([]byte, error) { + return c.DoEndpointLimit(ctx, baseURL, endpoint, apiKey, body, maxJSONResponseSize) +} + +// DoEndpointLimit is DoEndpoint with a response size limit; a response over +// the limit is an error rather than a truncated body. +func (c *Client) DoEndpointLimit(ctx context.Context, baseURL, endpoint, apiKey string, body map[string]any, limit int64) ([]byte, error) { jsonBody, err := json.Marshal(body) if err != nil { return nil, fmt.Errorf("failed to marshal request: %w", err) @@ -71,10 +80,13 @@ func (c *Client) DoEndpoint(ctx context.Context, baseURL, endpoint, apiKey strin resp.Body = providerstream.WithIdleTimeout(resp.Body, c.responseTimeout) defer resp.Body.Close() - respBody, err := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024)) + respBody, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) if err != nil { return nil, fmt.Errorf("failed to read provider response: %w", err) } + if int64(len(respBody)) > limit { + return nil, fmt.Errorf("provider response is larger than %d bytes", limit) + } if resp.StatusCode >= 400 { return nil, ParseProviderResponseError(resp, respBody) diff --git a/apps/gateway/internal/adapters/providers/openai/images.go b/apps/gateway/internal/adapters/providers/openai/images.go new file mode 100644 index 0000000..1a35e4e --- /dev/null +++ b/apps/gateway/internal/adapters/providers/openai/images.go @@ -0,0 +1,38 @@ +package openai + +import ( + "bytes" + "context" + "encoding/json" + + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" +) + +// maxImageResponseSize bounds an image generation response: a few base64 +// images, capped at 64 MiB. +const maxImageResponseSize = 64 << 20 + +// PrepareImageBody turns the client's image request into the provider's. +// It selects the provider's model and removes gateway-only fields. The H3 +// adapter then enforces the supported output options. +func PrepareImageBody(raw []byte, model domain.PublicModel) (map[string]any, error) { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() // Numbers are forwarded exactly as sent. + var body map[string]any + if err := dec.Decode(&body); err != nil || body == nil { + return nil, domain.ErrInvalidField("invalid JSON body") + } + body["model"] = model.UpstreamModelName + for _, field := range gatewayOnlyFields { + delete(body, field) + } + return body, nil +} + +// GenerateImages uses the supported MiniMax H3 generation/edit protocol. +func (a *Adapter) GenerateImages(ctx context.Context, raw []byte, model domain.PublicModel) ([]byte, error) { + if !model.SupportsImageEdits() { + return nil, domain.ErrUnsupportedFeature("unsupported image model: use MiniMax H3") + } + return a.generateMiniMaxH3Images(ctx, raw, model) +} diff --git a/apps/gateway/internal/adapters/providers/openai/images_minimax.go b/apps/gateway/internal/adapters/providers/openai/images_minimax.go new file mode 100644 index 0000000..b6a8776 --- /dev/null +++ b/apps/gateway/internal/adapters/providers/openai/images_minimax.go @@ -0,0 +1,65 @@ +package openai + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" +) + +// H3 has an OpenAI-shaped JSON API, including inline image references. +func (a *Adapter) generateMiniMaxH3Images(ctx context.Context, raw []byte, model domain.PublicModel) ([]byte, error) { + body, err := PrepareImageBody(raw, model) + if err != nil { + return nil, err + } + endpoint := "/images/generations" + if _, ok := body["images"]; ok { + endpoint = "/images/edits" + } + // A bounded 1K output keeps the initial UI price predictable. Other output + // sizes can be offered as separately priced options in a later version. + body["size"] = "auto" + body["area"] = "1K" + delete(body, "aspect_ratio") + body["output_format"] = "png" + body["response_format"] = "b64_json" + body["stream"] = false + var out []byte + _, err = a.proxy(ctx, body, nil, model, func(ctx context.Context, apiKey string, body map[string]any) (*http.Response, error) { + // H3 may wait 600 seconds before sending response headers. This dedicated + // transport does not impose chat's shorter header/idle deadline. + out, err = a.imageClient.DoEndpointLimit(ctx, model.ProviderConfig.BaseURL, endpoint, apiKey, body, maxImageResponseSize) + if err != nil { + return nil, err + } + return &http.Response{Body: io.NopCloser(bytes.NewReader(nil))}, nil + }) + if err != nil { + return nil, err + } + // Partial failure slots have an error rather than image bytes. Only retain + // successful images so billing and the UI count the actual outputs. + var response map[string]json.RawMessage + if err = json.Unmarshal(out, &response); err != nil { + return nil, err + } + var data []map[string]json.RawMessage + if err = json.Unmarshal(response["data"], &data); err != nil { + return nil, err + } + successful := make([]map[string]json.RawMessage, 0, len(data)) + for _, item := range data { + var b64 string + _ = json.Unmarshal(item["b64_json"], &b64) + if b64 != "" { + delete(item, "url") + successful = append(successful, item) + } + } + response["data"], _ = json.Marshal(successful) + return json.Marshal(response) +} diff --git a/apps/gateway/internal/adapters/providers/openai/images_minimax_test.go b/apps/gateway/internal/adapters/providers/openai/images_minimax_test.go new file mode 100644 index 0000000..9f08aaa --- /dev/null +++ b/apps/gateway/internal/adapters/providers/openai/images_minimax_test.go @@ -0,0 +1,78 @@ +package openai + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" +) + +func miniMaxModel(baseURL string) domain.PublicModel { + return domain.PublicModel{ + PublicModelID: "minimax/h3-image", UpstreamModelName: "MiniMax-H3-Image", SupportsImageGenerations: true, + ProviderConfig: domain.ProviderConfig{ProviderName: "minimax", BaseURL: baseURL, APIKeySecretRef: "MINIMAX_TEST_API_KEY"}, + } +} + +func TestMiniMaxH3EditsKeepsUsageAndOnlySuccessfulImages(t *testing.T) { + t.Setenv("MINIMAX_TEST_API_KEY", "k") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + if r.URL.Path != "/images/edits" || body["area"] != "1K" || body["response_format"] != "b64_json" || body["model"] != "MiniMax-H3-Image" { + t.Errorf("unexpected H3 request: %s %v", r.URL.Path, body) + } + if len(body["images"].([]any)) != 1 { + t.Error("lost reference image") + } + _, _ = w.Write([]byte(`{"data":[{"b64_json":"aW1n"},{"error":{"code":"1027"}}],"usage":{"generated_images":1,"output_tokens":8000,"input_tokens_details":{"text_tokens":200,"image_tokens":1000}}}`)) + })) + defer server.Close() + model := miniMaxModel(server.URL) + model.UpstreamModelName = "MiniMax-H3-Image" + model.SupportsImageGenerations = true + out, err := NewAdapter(time.Second).GenerateImages(context.Background(), []byte(`{"model":"public","prompt":"change background","images":[{"image_url":"data:image/png;base64,abc"}],"n":2}`), model) + if err != nil { + t.Fatal(err) + } + var got struct { + Data []any + Usage json.RawMessage + } + if json.Unmarshal(out, &got) != nil || len(got.Data) != 1 || len(got.Usage) == 0 { + t.Fatalf("bad response: %s", out) + } +} + +func TestMiniMaxH3GenerationWithoutReferences(t *testing.T) { + t.Setenv("MINIMAX_TEST_API_KEY", "k") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + } + if r.URL.Path != "/images/generations" || body["area"] != "1K" || body["model"] != "MiniMax-H3-Image" { + t.Errorf("unexpected generation: %s %v", r.URL.Path, body) + } + if _, ok := body["images"]; ok { + t.Error("text-to-image request gained references") + } + _, _ = w.Write([]byte(`{"data":[{"b64_json":"aW1n"}],"usage":{"generated_images":1}}`)) + })) + defer server.Close() + if _, err := NewAdapter(time.Second).GenerateImages(context.Background(), []byte(`{"model":"public","prompt":"a bird","n":1}`), miniMaxModel(server.URL)); err != nil { + t.Fatal(err) + } +} + +func TestImageAdapterRejectsUnsupportedModel(t *testing.T) { + model := miniMaxModel("http://unused") + model.UpstreamModelName = "unsupported-image-model" + if _, err := NewAdapter(time.Second).GenerateImages(context.Background(), []byte(`{"prompt":"a bird"}`), model); err == nil { + t.Fatal("unsupported model accepted") + } +} diff --git a/apps/gateway/internal/adapters/providers/openai/logging.go b/apps/gateway/internal/adapters/providers/openai/logging.go index e8c0eb9..5a632df 100644 --- a/apps/gateway/internal/adapters/providers/openai/logging.go +++ b/apps/gateway/internal/adapters/providers/openai/logging.go @@ -32,7 +32,7 @@ func (a *Adapter) logProviderRequest(ctx context.Context, model domain.PublicMod func summarizeProviderBody(body map[string]any) map[string]any { // Only fixed keys and numeric/boolean settings: arbitrary strings can contain prompts. summary := map[string]any{} - for _, key := range []string{"stream", "max_tokens", "max_completion_tokens", "temperature", "top_p", "presence_penalty", "frequency_penalty", "seed", "logprobs", "top_logprobs", "parallel_tool_calls", "store"} { + for _, key := range []string{"stream", "max_tokens", "max_completion_tokens", "temperature", "top_p", "presence_penalty", "frequency_penalty", "seed", "logprobs", "top_logprobs", "parallel_tool_calls", "store", "n"} { switch value := body[key].(type) { case bool, int, int32, int64, float32, float64: summary[key] = value diff --git a/apps/gateway/internal/application/ports/provider.go b/apps/gateway/internal/application/ports/provider.go index eab2dd1..2a20051 100644 --- a/apps/gateway/internal/application/ports/provider.go +++ b/apps/gateway/internal/application/ports/provider.go @@ -18,6 +18,14 @@ type Provider interface { Complete(ctx context.Context, raw []byte, model domain.PublicModel) ([]byte, *domain.TinfoilTransportProof, error) } +// ImageProvider generates images through the OpenAI images API. A provider +// adapter that doesn't implement it can't serve image models. +type ImageProvider interface { + // GenerateImages sends an images/generations request and returns the + // provider's JSON, with the images inline. + GenerateImages(ctx context.Context, raw []byte, model domain.PublicModel) ([]byte, error) +} + // ProviderStream is an upstream streaming response. type ProviderStream struct { Body io.ReadCloser diff --git a/apps/gateway/internal/application/ports/usage.go b/apps/gateway/internal/application/ports/usage.go index df62dd0..f1f8faa 100644 --- a/apps/gateway/internal/application/ports/usage.go +++ b/apps/gateway/internal/application/ports/usage.go @@ -20,3 +20,8 @@ type UsageMeter interface { type ReservationKeeper interface { KeepReservation(context.Context, string, func()) func() } + +// UsageReceiptMeter returns the authoritative charge receipt for image responses. +type UsageReceiptMeter interface { + CompleteWithReceipt(context.Context, string, domain.GenerateResult, domain.PublicModel, int64) (*int64, error) +} diff --git a/apps/gateway/internal/application/services/generate_service.go b/apps/gateway/internal/application/services/generate_service.go index 6db3396..a64b5a7 100644 --- a/apps/gateway/internal/application/services/generate_service.go +++ b/apps/gateway/internal/application/services/generate_service.go @@ -28,6 +28,10 @@ type GenerateService struct { logger ports.Logger // guestMaxOutputTokens is the output limit of guest chat requests. guestMaxOutputTokens int + // Image generation: see SetImageOptions and GenerateImages. + imageTimeout time.Duration + imageJobs *accountSlots + idempotency *idempotencyStore } func NewGenerateService( @@ -50,6 +54,10 @@ func NewGenerateService( logger: logger, guestMaxOutputTokens: DefaultGuestMaxOutputTokens, + + imageTimeout: DefaultImageTimeout, + imageJobs: newAccountSlots(DefaultImageConcurrency), + idempotency: newIdempotencyStore(), } } diff --git a/apps/gateway/internal/application/services/idempotency.go b/apps/gateway/internal/application/services/idempotency.go new file mode 100644 index 0000000..c0acf8f --- /dev/null +++ b/apps/gateway/internal/application/services/idempotency.go @@ -0,0 +1,188 @@ +package services + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "sync" + "time" + + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" +) + +// Idempotency-Key handling for requests that charge per result (image +// generation). Like Stripe's: the first request with a key runs; a retry +// while it runs is refused; a retry after it succeeded gets the same result +// without running or charging again. A key is bound to the account and the +// request body that first used it. A request that failed was not charged, so +// its key can be retried. +// +// Results are kept in memory only, for a short time and up to a size limit: +// the gateway stores no media. After that a retry with the key is refused +// rather than charged twice, for as long as the key is remembered. +const ( + idempotencyReplayTTL = 15 * time.Minute + idempotencyKeyTTL = 24 * time.Hour + idempotencyReplayBytes = 256 << 20 + maxIdempotencyKeyLen = 255 +) + +type idempotencyStore struct { + mu sync.Mutex + entries map[idempotencyKey]*idempotentRequest + replayBytes int64 + now func() time.Time +} + +// Retry keys are scoped to an authenticated account, not used as credentials. +// Keep the two fields separate so lookup compares the exact pair. +type idempotencyKey struct { + accountID string + retryKey string +} + +type idempotentRequest struct { + key idempotencyKey + bodyHash [32]byte + done bool + result []byte // Replayable result, while kept. + replayAt time.Time // Until when result is kept. + expires time.Time // Until when the key is remembered. +} + +func newIdempotencyStore() *idempotencyStore { + return &idempotencyStore{entries: map[idempotencyKey]*idempotentRequest{}, now: time.Now} +} + +// begin claims key for a request. It returns the stored result when the key +// already succeeded, and an error when the key is in use or was used for a +// different request. +func (s *idempotencyStore) begin(accountID, key string, body []byte) (*idempotentRequest, []byte, error) { + if len(key) > maxIdempotencyKeyLen { + return nil, nil, domain.ErrInvalidField("Idempotency-Key must be at most 255 characters") + } + id := idempotencyKey{accountID: accountID, retryKey: key} + bodyHash := sha256.Sum256(canonicalJSON(body)) + s.mu.Lock() + defer s.mu.Unlock() + now := s.now() + s.expire(now) + if entry, ok := s.entries[id]; ok { + switch { + case entry.bodyHash != bodyHash: + return nil, nil, domain.ErrIdempotencyKeyReused("This Idempotency-Key was already used for a different request.") + case !entry.done: + return nil, nil, domain.ErrIdempotencyInProgress() + case entry.result != nil: + return nil, entry.result, nil + default: + return nil, nil, domain.ErrIdempotencyKeyReused("This Idempotency-Key was already used for a finished request whose result is no longer kept.") + } + } + entry := &idempotentRequest{key: id, bodyHash: bodyHash, expires: now.Add(idempotencyKeyTTL)} + s.entries[id] = entry + return entry, nil, nil +} + +// finish records a successful request and keeps its result for replay while +// it fits. +func (s *idempotencyStore) finish(entry *idempotentRequest, result []byte) { + s.mu.Lock() + defer s.mu.Unlock() + now := s.now() + entry.done = true + entry.expires = now.Add(idempotencyKeyTTL) + s.expire(now) + if int64(len(result)) > idempotencyReplayBytes/4 { + return + } + for s.replayBytes+int64(len(result)) > idempotencyReplayBytes && s.dropOldestResult() { + } + entry.result = result + entry.replayAt = now.Add(idempotencyReplayTTL) + s.replayBytes += int64(len(result)) +} + +// abandon releases a key whose request failed, so it can be retried. +func (s *idempotencyStore) abandon(entry *idempotentRequest) { + s.mu.Lock() + defer s.mu.Unlock() + if s.entries[entry.key] == entry { + delete(s.entries, entry.key) + } +} + +func (s *idempotencyStore) expire(now time.Time) { + for id, entry := range s.entries { + if entry.result != nil && now.After(entry.replayAt) { + s.replayBytes -= int64(len(entry.result)) + entry.result = nil + } + if entry.done && now.After(entry.expires) { + delete(s.entries, id) + } + } +} + +func (s *idempotencyStore) dropOldestResult() bool { + var oldest *idempotentRequest + for _, entry := range s.entries { + if entry.result != nil && (oldest == nil || entry.replayAt.Before(oldest.replayAt)) { + oldest = entry + } + } + if oldest == nil { + return false + } + s.replayBytes -= int64(len(oldest.result)) + oldest.result = nil + return true +} + +// canonicalJSON is body with its object keys sorted and whitespace removed, +// so the same request sent by another client library matches its key. A +// body that isn't JSON is compared as it is. +func canonicalJSON(body []byte) []byte { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var v any + if dec.Decode(&v) != nil { + return body + } + out, err := json.Marshal(v) + if err != nil { + return body + } + return out +} + +// accountSlots limits how many slow jobs one account runs at once. +type accountSlots struct { + limit int + mu sync.Mutex + running map[string]int +} + +func newAccountSlots(limit int) *accountSlots { + return &accountSlots{limit: limit, running: map[string]int{}} +} + +// acquire takes a slot for account; release returns it. +func (a *accountSlots) acquire(account string) (release func(), ok bool) { + a.mu.Lock() + defer a.mu.Unlock() + if a.running[account] >= a.limit { + return nil, false + } + a.running[account]++ + var once sync.Once + return func() { + once.Do(func() { + a.mu.Lock() + defer a.mu.Unlock() + if a.running[account]--; a.running[account] <= 0 { + delete(a.running, account) + } + }) + }, true +} diff --git a/apps/gateway/internal/application/services/image_generations.go b/apps/gateway/internal/application/services/image_generations.go new file mode 100644 index 0000000..b7a5bf3 --- /dev/null +++ b/apps/gateway/internal/application/services/image_generations.go @@ -0,0 +1,307 @@ +package services + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/adapters/http/middleware" + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/adapters/observability/metrics" + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/application/ports" + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" + "github.com/google/uuid" +) + +// Image generation defaults; see SetImageOptions. +const ( + // DefaultImageTimeout allows MiniMax's 600-second synchronous wait. + DefaultImageTimeout = 660 * time.Second + // DefaultImageConcurrency is how many image generations one account + // runs at once. + DefaultImageConcurrency = 2 +) + +// ImageCall is a finished image generation: the provider's JSON, without the +// provider's own usage and cost. +type ImageCall struct { + Model domain.PublicModel + Body []byte + // Replayed is set when the body is the stored result of an earlier + // request with the same Idempotency-Key; nothing was charged again. + Replayed bool +} + +// SetImageOptions sets the image generation timeout and how many image +// generations one account may run at once. +func (s *GenerateService) SetImageOptions(timeout time.Duration, perAccount int) { + if timeout > 0 { + s.imageTimeout = timeout + } + if perAccount > 0 { + s.imageJobs = newAccountSlots(perAccount) + } +} + +// GenerateImages runs POST /v1/images/generations. It authenticates, keeps +// guests out, holds the price of the requested images, calls the provider +// once without automatic retry or fallback, and charges successful outputs. A +// request the provider refuses costs nothing. +// +// idempotencyKey, when set, makes retries safe: a retry of a finished request +// gets the same images back without a second charge, and a retry of a +// running one is refused until it finishes. +func (s *GenerateService) GenerateImages(ctx context.Context, raw []byte, req domain.GenerateRequest, bearerToken, idempotencyKey string) (*ImageCall, error) { + const endpoint = domain.EndpointImageGenerations + start := time.Now() + requestID := middleware.GetRequestID(ctx) + fail := func(err error, model *domain.PublicModel) error { + err = clientGone(ctx, err) + s.recordTerminalOutcome(metrics.OutcomeError, endpoint, req, model, time.Since(start).Milliseconds()) + s.recordError(endpoint, req, model, metrics.PhaseRequest, err) + return err + } + + authCtx, err := s.auth.AuthenticateAPIKey(ctx, bearerToken) + if err != nil { + return nil, fail(err, nil) + } + reject := func(err error, model *domain.PublicModel) error { + err = fail(err, model) + if model == nil { + return domain.Annotate(err, "account_id", authCtx.Account.ID, "requested_model", req.PublicModelID) + } + return domain.Annotate(err, "account_id", authCtx.Account.ID, "model", model.PublicModelID, "provider", model.ProviderConfig.ProviderName) + } + // Guests never get media. + if authCtx.Account.IsGuest { + return nil, reject(domain.ErrGuestModelNotAllowed(req.PublicModelID), nil) + } + model, err := s.catalog.GetPublicModel(ctx, req.PublicModelID) + if err != nil { + return nil, reject(err, nil) + } + if err := s.validateRequest(endpoint, req, model); err != nil { + s.recordFailure(ctx, nil, &authCtx, endpoint, &req, &model, err, nil, time.Since(start).Milliseconds()) + return nil, reject(err, &model) + } + + if !strings.EqualFold(model.ProviderConfig.ProviderName, "minimax") { + return nil, reject(domain.ErrUnsupportedFeature("only MiniMax image providers are enabled"), &model) + } + if req.ImageEdit && !model.SupportsImageEdits() { + return nil, reject(domain.ErrUnsupportedFeature("this model does not support image editing"), &model) + } + var idem *idempotentRequest + if idempotencyKey != "" { + var replay []byte + idem, replay, err = s.idempotency.begin(authCtx.Account.ID, idempotencyKey, raw) + if err != nil { + return nil, reject(err, &model) + } + if replay != nil { + s.logger.Info("image generation replayed", "request_id", requestID, "account_id", authCtx.Account.ID, "model", model.PublicModelID) + return &ImageCall{Model: model, Body: replay, Replayed: true}, nil + } + } + call, err := s.generateImages(ctx, authCtx, raw, req, model, start) + if idem != nil { + if err != nil { + s.idempotency.finish(idem, nil) // Never resubmit a possibly billed upstream request with this key. + } else { + s.idempotency.finish(idem, call.Body) + } + } + return call, err +} + +func (s *GenerateService) generateImages(ctx context.Context, authCtx domain.AuthContext, raw []byte, req domain.GenerateRequest, model domain.PublicModel, start time.Time) (*ImageCall, error) { + const endpoint = domain.EndpointImageGenerations + requestID := middleware.GetRequestID(ctx) + reject := func(err error) error { + err = clientGone(ctx, err) + s.recordTerminalOutcome(metrics.OutcomeError, endpoint, req, &model, time.Since(start).Milliseconds()) + s.recordError(endpoint, req, &model, metrics.PhaseRequest, err) + return domain.Annotate(err, "account_id", authCtx.Account.ID, "model", model.PublicModelID, "provider", model.ProviderConfig.ProviderName) + } + + release, ok := s.imageJobs.acquire(authCtx.Account.ID) + if !ok { + return nil, reject(domain.ErrTooManyConcurrent(s.imageJobs.limit)) + } + defer release() + + masked, _, err := s.maskBody(ctx, raw, authCtx.APIKey.PIIMode, endpoint) + if err != nil { + s.recordFailure(ctx, nil, &authCtx, endpoint, &req, &model, err, nil, time.Since(start).Milliseconds()) + return nil, reject(err) + } + reservationID, err := s.metering.Reserve(ctx, authCtx, endpoint, req, model, uuid.NewString()) + if err != nil { + return nil, reject(err) + } + + // From here the provider runs and bills us: like chat, the generation no + // longer follows the client's connection, so a client that hangs up is + // still charged for the images it asked for, and gets them on a retry + // with the same Idempotency-Key. + ctx, cancelTimeout := context.WithTimeout(context.WithoutCancel(ctx), s.imageTimeout) + defer cancelTimeout() + ctx, cancelInference := context.WithCancelCause(ctx) + defer cancelInference(nil) + stopRenewal := func() {} + if keeper, ok := s.metering.(ports.ReservationKeeper); ok { + stopRenewal = keeper.KeepReservation(ctx, reservationID, func() { + cancelInference(domain.ErrInternal("credit reservation could not be maintained").WithMeta("dependency", "metering", "cause", domain.CauseCreditHoldLost)) + }) + } + defer stopRenewal() + + var providerBody []byte + attempt := func(target domain.PublicModel) ([]byte, int64, error) { + if err := s.validateRequest(endpoint, req, target); err != nil { + return nil, 0, err + } + provider, err := s.registry.GetProvider(target.ProviderConfig.ProviderName) + if err != nil { + return nil, 0, domain.ErrProviderUnavailable(target.ProviderConfig.ProviderName).WithMeta( + "fault", domain.FaultNexus, "cause", domain.CauseProviderNotRegistered) + } + images, ok := provider.(ports.ImageProvider) + if !ok { + return nil, 0, domain.ErrUnsupportedEndpoint(target.PublicModelID, endpoint).WithMeta( + "fault", domain.FaultNexus, "provider", target.ProviderConfig.ProviderName) + } + upstreamStart := time.Now() + body, err := images.GenerateImages(ctx, masked, target) + recordUpstreamLatency(req.PublicModelID, target.ProviderConfig.ProviderName, upstreamStart, err) + if err != nil { + return nil, 0, imageProviderError(err) + } + providerBody = body + return readImageResponse(body, req.Images) + } + + executed := model + body, count, err := attempt(model) + + latencyMs := time.Since(start).Milliseconds() + if err != nil { + if cause, ok := context.Cause(ctx).(*domain.GatewayError); ok { + err = cause + } else if errors.Is(ctx.Err(), context.DeadlineExceeded) { + err = domain.ErrProviderTimeout(executed.ProviderConfig.ProviderName).WithMeta("timeout_ms", s.imageTimeout.Milliseconds()) + } + s.logger.Error("provider request failed", s.buildErrorLogFields(ctx, requestID, &authCtx, endpoint, req.PublicModelID, executed, err, latencyMs)...) + s.recordGeneration(metrics.OutcomeError, endpoint, req, executed, latencyMs) + s.recordError(endpoint, req, &executed, metrics.PhaseProvider, err) + // No usage: a failed or refused generation is not charged. + s.recordFailure(ctx, &reservationID, &authCtx, endpoint, &req, &executed, err, nil, latencyMs) + return nil, err + } + + var upstream struct { + Usage json.RawMessage `json:"usage"` + } + _ = json.Unmarshal(providerBody, &upstream) + normalized, ok := domain.ParseImageTokenUsage(upstream.Usage, count) + if !ok { + err := domain.ErrProviderError(http.StatusBadGateway, "provider returned missing or inconsistent image token usage") + s.recordFailure(ctx, &reservationID, &authCtx, endpoint, &req, &model, err, nil, latencyMs) + return nil, err + } + usage := &normalized + result := domain.GenerateResult{ + PublicModelID: executed.PublicModelID, ProviderName: executed.ProviderConfig.ProviderName, + ProviderModelID: executed.ProviderModelID, Usage: usage, + } + metrics.RecordUsage(usage, req.PublicModelID, executed.ProviderConfig.ProviderName) + s.recordGeneration(metrics.OutcomeSuccess, endpoint, req, executed, latencyMs) + s.logger.Info("generation completed", + "request_id", requestID, "reservation_id", reservationID, "account_id", authCtx.Account.ID, + "endpoint", endpoint, "model", req.PublicModelID, "provider", executed.ProviderConfig.ProviderName, + "provider_model", executed.UpstreamModelName, "images_requested", req.Images, "images", count, + "latency_ms", latencyMs, "gateway_status", 200, "upstream_status", 200) + var charged *int64 + if receipts, ok := s.metering.(ports.UsageReceiptMeter); ok { + charged, err = receipts.CompleteWithReceipt(context.WithoutCancel(ctx), reservationID, result, executed, latencyMs) + } else { + err = s.metering.RecordSuccess(context.WithoutCancel(ctx), reservationID, authCtx, endpoint, req, result, executed, latencyMs) + } + if err != nil { + s.logger.Error("failed to record usage", "request_id", requestID, "reservation_id", reservationID, "error_type", fmt.Sprintf("%T", err)) + } + if charged != nil { + var response map[string]json.RawMessage + if json.Unmarshal(body, &response) == nil { + response["nexus_billing"], _ = json.Marshal(map[string]any{"charged_cents": float64(*charged) / 1_000_000, "currency": "EUR"}) + body, _ = json.Marshal(response) + } + } + return &ImageCall{Model: executed, Body: body}, nil +} + +// readImageResponse checks a provider's image response and counts its +// images. Every image must be inline: the gateway stores no media and never +// hands out a provider's URL. The provider's usage and cost are removed; +// what the account pays is Nexus's price. +func readImageResponse(body []byte, requested *int) ([]byte, int64, error) { + var resp map[string]json.RawMessage + if err := json.Unmarshal(body, &resp); err != nil { + return nil, 0, domain.ErrProviderError(http.StatusBadGateway, "provider returned an invalid response") + } + var data []struct { + B64JSON string `json:"b64_json"` + } + if err := json.Unmarshal(resp["data"], &data); err != nil || len(data) == 0 { + return nil, 0, domain.ErrProviderError(http.StatusBadGateway, "provider returned no images") + } + for _, item := range data { + if item.B64JSON == "" { + return nil, 0, domain.ErrProviderError(http.StatusBadGateway, "provider returned an image that is not inline") + } + } + if requested != nil && len(data) > *requested { + return nil, 0, domain.ErrProviderError(http.StatusBadGateway, "provider returned more images than requested") + } + delete(resp, "usage") + delete(resp, "nexus_billing") + out, err := json.Marshal(resp) + if err != nil { + return nil, 0, domain.ErrInternal("an internal error occurred").WithMeta("reason", "re-encode image response") + } + return out, int64(len(data)), nil +} + +// imageProviderError turns a provider's content-policy refusal into a 422 +// the client can act on. Providers name it differently: OpenAI says +// moderation_blocked or content_policy_violation; others say safety, nsfw or +// content filter. +func imageProviderError(err error) error { + var gwErr *domain.GatewayError + if !errors.As(err, &gwErr) { + return err + } + status, _ := gwErr.Metadata["upstream_status"].(int) + if status != http.StatusBadRequest && status != http.StatusUnprocessableEntity { + return err + } + var text strings.Builder + for _, key := range []string{"upstream_code", "upstream_type", "upstream_reason", "upstream_error"} { + if v, ok := gwErr.Metadata[key].(string); ok { + text.WriteString(strings.ToLower(v)) + text.WriteByte(' ') + } + } + for _, marker := range []string{"moderation", "content_policy", "content policy", "content_filter", "content filter", "safety", "nsfw"} { + if strings.Contains(text.String(), marker) { + refusal := domain.ErrContentPolicy() + refusal.Metadata = gwErr.Metadata + return refusal + } + } + return err +} diff --git a/apps/gateway/internal/application/services/image_generations_test.go b/apps/gateway/internal/application/services/image_generations_test.go new file mode 100644 index 0000000..7dd7355 --- /dev/null +++ b/apps/gateway/internal/application/services/image_generations_test.go @@ -0,0 +1,435 @@ +package services + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/application/ports" + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" + "github.com/shopspring/decimal" +) + +const twoImages = `{"created":1,"data":[{"b64_json":"aGVsbG8="},{"b64_json":"d29ybGQ="}],"usage":{"input_tokens":10,"output_tokens":200,"total_tokens":210,"generated_images":2,"input_tokens_details":{"text_tokens":10,"image_tokens":0},"output_tokens_details":{"image_tokens":200}}}` + +// imageProvider is an OpenAI-compatible provider that also generates images. +type imageProvider struct { + stubProvider + mu sync.Mutex + body string + imageErr error + imageRuns int + lastImage string + block chan struct{} // When set, a generation waits for it. +} + +func (p *imageProvider) GenerateImages(_ context.Context, raw []byte, _ domain.PublicModel) ([]byte, error) { + p.mu.Lock() + p.imageRuns++ + p.lastImage = string(raw) + block := p.block + p.mu.Unlock() + if block != nil { + <-block + } + if p.imageErr != nil { + return nil, p.imageErr + } + return []byte(p.body), nil +} + +// imageMeter records what each image generation was charged for. +type imageMeter struct { + stubUsageMeter + mu sync.Mutex + charged []domain.Usage +} + +func (m *imageMeter) RecordSuccess(ctx context.Context, reservationID string, auth domain.AuthContext, endpoint string, req domain.GenerateRequest, result domain.GenerateResult, model domain.PublicModel, latencyMs int64) error { + m.mu.Lock() + if result.Usage != nil { + m.charged = append(m.charged, *result.Usage) + } + m.mu.Unlock() + return m.stubUsageMeter.RecordSuccess(ctx, reservationID, auth, endpoint, req, result, model, latencyMs) +} + +func imageModel() domain.PublicModel { + price := decimal.RequireFromString("0.02") + return domain.PublicModel{ + PublicModelID: "minimax/h3-image", + ProviderModelID: "minimax-h3-image", + UpstreamModelName: "MiniMax-H3-Image", + ProviderConfig: domain.ProviderConfig{ProviderName: "minimax"}, + SupportsImageGenerations: true, + ImageInputPricePerMillion: &price, ImageMaxCostPerImage: &price, + } +} + +func newImageService(meter ports.UsageMeter, provider ports.Provider, guest bool) *GenerateService { + return NewGenerateService( + &stubAuthService{authCtx: domain.AuthContext{ + Account: domain.Account{ID: "acc1", Status: domain.AccountStatusActive, IsGuest: guest}, + APIKey: domain.APIKey{ID: "key1", Active: true}, + }}, + &stubModelCatalog{model: imageModel()}, + nil, + &stubProviderRegistry{providers: map[string]ports.Provider{"minimax": provider}}, + meter, + nil, + stubLogger{}, + ) +} + +func imageRequest(n int) domain.GenerateRequest { + return domain.GenerateRequest{PublicModelID: "minimax/h3-image", RequestedModelID: "minimax/h3-image", Images: &n} +} + +const imageBody = `{"model":"minimax/h3-image","prompt":"a fox","n":2}` + +func TestGenerateImagesChargesReturnedImages(t *testing.T) { + meter := &imageMeter{} + provider := &imageProvider{body: twoImages} + svc := newImageService(meter, provider, false) + + call, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(2), "sk-test", "") + if err != nil { + t.Fatalf("GenerateImages: %v", err) + } + if meter.reserveCalls != 1 || *meter.lastReserveReq.Images != 2 { + t.Fatalf("reserve calls = %d with images %v, want one hold for 2", meter.reserveCalls, meter.lastReserveReq.Images) + } + if len(meter.charged) != 1 || meter.charged[0].Images != 2 { + t.Fatalf("charged %+v, want 2 images once", meter.charged) + } + var out map[string]json.RawMessage + if err := json.Unmarshal(call.Body, &out); err != nil { + t.Fatalf("response is not JSON: %v", err) + } + if _, ok := out["usage"]; ok { + t.Fatal("the provider's usage and cost must not reach the client") + } + if !strings.Contains(string(call.Body), "aGVsbG8=") { + t.Fatal("images missing from the response") + } + if !strings.Contains(provider.lastImage, `"prompt":"a fox"`) { + t.Fatalf("provider got %s", provider.lastImage) + } +} + +func TestGenerateImagesRefusedPromptCostsNothing(t *testing.T) { + meter := &imageMeter{} + refusal := domain.ErrProviderError(400, "provider rejected request").WithMeta( + "upstream_status", 400, "upstream_code", "moderation_blocked") + svc := newImageService(meter, &imageProvider{imageErr: refusal}, false) + + _, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(1), "sk-test", "") + var gwErr *domain.GatewayError + if !errors.As(err, &gwErr) || gwErr.HTTPStatus != 422 || gwErr.Code != domain.ErrCodeContentPolicy { + t.Fatalf("err = %v, want a 422 content_policy_violation", err) + } + if len(meter.charged) != 0 || meter.failureCalls != 1 { + t.Fatalf("charged %+v with %d failures, want nothing charged and the hold released", meter.charged, meter.failureCalls) + } +} + +func TestGenerateImagesRejectsGuests(t *testing.T) { + meter := &imageMeter{} + provider := &imageProvider{body: twoImages} + svc := newImageService(meter, provider, true) + + _, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(1), "sk-guest", "") + var gwErr *domain.GatewayError + if !errors.As(err, &gwErr) || gwErr.HTTPStatus != 403 { + t.Fatalf("err = %v, want 403 for a guest", err) + } + if meter.reserveCalls != 0 || provider.imageRuns != 0 { + t.Fatal("a guest's image request must not reach metering or the provider") + } +} + +func TestGenerateImagesRejectsChatModel(t *testing.T) { + meter := &imageMeter{} + svc := newDirectModelGenerateService(&meter.stubUsageMeter, &stubProvider{}) + _, err := svc.GenerateImages(context.Background(), []byte(`{"model":"model","prompt":"x"}`), domain.GenerateRequest{PublicModelID: "model"}, "sk", "") + var gwErr *domain.GatewayError + if !errors.As(err, &gwErr) || gwErr.Code != domain.ErrCodeUnsupportedEndpoint { + t.Fatalf("err = %v, want unsupported_endpoint for a chat model", err) + } +} + +func TestGenerateImagesIdempotencyKeyReplaysWithoutCharging(t *testing.T) { + meter := &imageMeter{} + provider := &imageProvider{body: twoImages} + svc := newImageService(meter, provider, false) + + first, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(2), "sk", "key-1") + if err != nil { + t.Fatal(err) + } + second, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(2), "sk", "key-1") + if err != nil { + t.Fatal(err) + } + if !second.Replayed || string(second.Body) != string(first.Body) { + t.Fatal("a retry with the same key must replay the first result") + } + if provider.imageRuns != 1 || len(meter.charged) != 1 { + t.Fatalf("provider ran %d times and charged %d times, want once each", provider.imageRuns, len(meter.charged)) + } + + _, err = svc.GenerateImages(context.Background(), []byte(`{"model":"minimax/h3-image","prompt":"a cat"}`), imageRequest(1), "sk", "key-1") + var gwErr *domain.GatewayError + if !errors.As(err, &gwErr) || gwErr.Code != domain.ErrCodeIdempotencyKeyReused { + t.Fatalf("err = %v, want idempotency_key_reused for a different body", err) + } +} + +func TestGenerateImagesIdempotencyKeyInProgress(t *testing.T) { + meter := &imageMeter{} + provider := &imageProvider{body: twoImages, block: make(chan struct{})} + svc := newImageService(meter, provider, false) + + done := make(chan error, 1) + go func() { + _, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(2), "sk", "key-2") + done <- err + }() + waitFor(t, func() bool { provider.mu.Lock(); defer provider.mu.Unlock(); return provider.imageRuns == 1 }) + + _, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(2), "sk", "key-2") + var gwErr *domain.GatewayError + if !errors.As(err, &gwErr) || gwErr.Code != domain.ErrCodeIdempotencyInProgress || gwErr.HTTPStatus != 409 { + t.Fatalf("err = %v, want 409 idempotency_request_in_progress", err) + } + close(provider.block) + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestGenerateImagesFailedKeyCannotResubmitPossiblyBilledJob(t *testing.T) { + meter := &imageMeter{} + provider := &imageProvider{imageErr: domain.ErrProviderUnavailable("minimax")} + svc := newImageService(meter, provider, false) + if _, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(2), "sk", "key-3"); err == nil { + t.Fatal("want provider error") + } + provider.imageErr = nil + provider.body = twoImages + if _, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(2), "sk", "key-3"); err == nil { + t.Fatal("must not resubmit ambiguous request") + } + if provider.imageRuns != 1 || len(meter.charged) != 0 { + t.Fatal("retry submitted or charged") + } +} + +func TestGenerateImagesLimitsConcurrentJobsPerAccount(t *testing.T) { + meter := &imageMeter{} + provider := &imageProvider{body: twoImages, block: make(chan struct{})} + svc := newImageService(meter, provider, false) + svc.SetImageOptions(0, 1) + + done := make(chan error, 1) + go func() { + _, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(2), "sk", "") + done <- err + }() + waitFor(t, func() bool { provider.mu.Lock(); defer provider.mu.Unlock(); return provider.imageRuns == 1 }) + + _, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(2), "sk", "") + var gwErr *domain.GatewayError + if !errors.As(err, &gwErr) || gwErr.HTTPStatus != 429 || gwErr.Code != domain.ErrCodeTooManyConcurrent { + t.Fatalf("err = %v, want 429 too_many_concurrent_requests", err) + } + if meter.reserveCalls != 1 { + t.Fatalf("reserve calls = %d, want the refused request never to hold credit", meter.reserveCalls) + } + close(provider.block) + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestGenerateImagesTimesOut(t *testing.T) { + meter := &imageMeter{} + provider := &imageProvider{body: twoImages, block: make(chan struct{})} + defer close(provider.block) + svc := newImageService(meter, &timeoutAwareImageProvider{provider}, false) + svc.SetImageOptions(50*time.Millisecond, 0) + + _, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(2), "sk", "") + var gwErr *domain.GatewayError + if !errors.As(err, &gwErr) || gwErr.Code != domain.ErrCodeProviderTimeout { + t.Fatalf("err = %v, want provider_timeout", err) + } + if len(meter.charged) != 0 || meter.failureCalls != 1 { + t.Fatal("a timed-out generation must release its hold uncharged") + } +} + +// timeoutAwareImageProvider returns when the context ends, like a real HTTP +// call would. +type timeoutAwareImageProvider struct{ *imageProvider } + +func (p *timeoutAwareImageProvider) GenerateImages(ctx context.Context, raw []byte, model domain.PublicModel) ([]byte, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +func TestReadImageResponseRejectsURLsAndExtras(t *testing.T) { + one := 1 + for name, body := range map[string]string{ + "url only": `{"data":[{"url":"https://cdn.example/x.png"}]}`, + "no images": `{"data":[]}`, + "not json": `nope`, + "too many": `{"data":[{"b64_json":"a"},{"b64_json":"b"}]}`, + "empty item": `{"data":[{"b64_json":""}]}`, + } { + if _, _, err := readImageResponse([]byte(body), &one); err == nil { + t.Errorf("%s: want an error", name) + } + } +} + +func TestImageProviderErrorKeepsOtherRejections(t *testing.T) { + plain := domain.ErrProviderError(400, "bad size").WithMeta("upstream_status", 400, "upstream_code", "invalid_parameter") + var gwErr *domain.GatewayError + if !errors.As(imageProviderError(plain), &gwErr) || gwErr.Code == domain.ErrCodeContentPolicy { + t.Fatal("an ordinary 400 is not a content-policy refusal") + } + server := domain.ErrProviderError(500, "safety system down").WithMeta("upstream_status", 500, "upstream_error", "safety system down") + if errors.As(imageProviderError(server), &gwErr) && gwErr.Code == domain.ErrCodeContentPolicy { + t.Fatal("a provider outage is not a content-policy refusal") + } +} + +func TestIdempotencyStoreForgetsReplayAfterTTL(t *testing.T) { + now := time.Unix(0, 0) + store := newIdempotencyStore() + store.now = func() time.Time { return now } + entry, _, err := store.begin("acc", "k", []byte("body")) + if err != nil { + t.Fatal(err) + } + store.finish(entry, []byte("result")) + if _, replay, _ := store.begin("acc", "k", []byte("body")); string(replay) != "result" { + t.Fatal("want the stored result") + } + if _, _, err := store.begin("other", "k", []byte("body")); err != nil { + t.Fatalf("keys are per account: %v", err) + } + now = now.Add(idempotencyReplayTTL + time.Second) + _, replay, err := store.begin("acc", "k", []byte("body")) + var gwErr *domain.GatewayError + if replay != nil || !errors.As(err, &gwErr) || gwErr.Code != domain.ErrCodeIdempotencyKeyReused { + t.Fatalf("after the replay TTL: replay=%q err=%v, want a refusal, never a second run", replay, err) + } + now = now.Add(idempotencyKeyTTL) + if _, _, err := store.begin("acc", "k", []byte("body")); err != nil { + t.Fatalf("an expired key is free again: %v", err) + } +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatal("timed out waiting") + } + time.Sleep(5 * time.Millisecond) + } +} + +// Another client library serializes the same request differently; it is still +// the same request. +func TestIdempotencyKeyMatchesTheSameRequestFormattedDifferently(t *testing.T) { + store := newIdempotencyStore() + entry, _, err := store.begin("acc", "k", []byte(`{"model":"m","prompt":"a fox","n":2}`)) + if err != nil { + t.Fatal(err) + } + store.finish(entry, []byte("result")) + if _, replay, err := store.begin("acc", "k", []byte("{\n \"n\": 2, \"prompt\": \"a fox\",\n \"model\": \"m\"\n}")); err != nil || string(replay) != "result" { + t.Fatalf("replay=%q err=%v, want the stored result", replay, err) + } + if _, _, err := store.begin("acc", "k", []byte(`{"model":"m","prompt":"a fox","n":3}`)); err == nil { + t.Fatal("a different n is a different request") + } +} + +func TestGenerateImagesRejectsOtherProvidersAndUnsupportedModels(t *testing.T) { + for _, tc := range []struct { + name, provider string + edit bool + }{ + {"other provider", "phala", false}, {"unsupported model", "minimax", true}, + } { + t.Run(tc.name, func(t *testing.T) { + meter := &imageMeter{} + provider := &imageProvider{body: twoImages} + svc := newImageService(meter, provider, false) + model := imageModel() + model.ProviderConfig.ProviderName = tc.provider + if tc.edit { + model.UpstreamModelName = "unsupported-image-model" + } + svc.catalog = &stubModelCatalog{model: model} + req := imageRequest(1) + req.ImageEdit = tc.edit + _, err := svc.GenerateImages(context.Background(), []byte(imageBody), req, "sk-test", "") + if err == nil || meter.reserveCalls != 0 || provider.imageRuns != 0 { + t.Fatalf("request not rejected before billing: %v", err) + } + }) + } +} + +type imageReceiptMeter struct{ imageMeter } + +func (m *imageReceiptMeter) CompleteWithReceipt(ctx context.Context, id string, result domain.GenerateResult, model domain.PublicModel, ms int64) (*int64, error) { + err := m.RecordSuccess(ctx, id, domain.AuthContext{}, domain.EndpointImageGenerations, domain.GenerateRequest{}, result, model, ms) + charged := int64(4_300_000) + return &charged, err +} + +func TestGenerateH3ImagesBillsSeparateTokenDimensionsAndReplaysReceipt(t *testing.T) { + meter := &imageReceiptMeter{} + provider := &imageProvider{body: `{"data":[{"b64_json":"aGVsbG8="}],"usage":{"input_tokens":3000,"output_tokens":4000,"total_tokens":7000,"generated_images":1,"input_tokens_details":{"text_tokens":1000,"image_tokens":2000},"output_tokens_details":{"image_tokens":4000}}}`} + svc := newImageService(meter, provider, false) + model := imageModel() + model.UpstreamModelName = "MiniMax-H3-Image" + reference, ceiling := decimal.RequireFromString("3"), decimal.RequireFromString("0.50") + model.ImageInputPricePerMillion = &reference + model.ImageMaxCostPerImage = &ceiling + svc.catalog = &stubModelCatalog{model: model} + for i := 0; i < 2; i++ { + call, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(1), "sk-test", "token-receipt") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(call.Body), `"charged_cents":4.3`) || strings.Contains(string(call.Body), `"usage"`) { + t.Fatalf("response=%s", call.Body) + } + } + if len(meter.charged) != 1 { + t.Fatalf("charged %d times", len(meter.charged)) + } + usage := meter.charged[0] + if usage.PromptTokens != 1000 || usage.ImageInputTokens != 2000 || usage.CompletionTokens != 4000 { + t.Fatalf("usage=%+v", usage) + } + provider.body = `{"data":[{"b64_json":"aGVsbG8="}],"usage":{}}` + if _, err := svc.GenerateImages(context.Background(), []byte(imageBody), imageRequest(1), "sk-test", "bad-usage"); err == nil { + t.Fatal("accepted missing token usage") + } + if len(meter.charged) != 1 { + t.Fatal("malformed usage charged the user") + } +} diff --git a/apps/gateway/internal/application/services/pii.go b/apps/gateway/internal/application/services/pii.go index 155614e..e80451b 100644 --- a/apps/gateway/internal/application/services/pii.go +++ b/apps/gateway/internal/application/services/pii.go @@ -105,6 +105,21 @@ func (m *requestPIIMasker) maskBody(body map[string]any, endpoint string) error } return nil } + if endpoint == domain.EndpointImageGenerations { + // The prompt is drawn, not answered: placeholders stay in the image. + for _, field := range []string{"prompt", "user"} { + text, ok := body[field].(string) + if !ok { + continue + } + masked, err := m.maskText(field, text) + if err != nil { + return err + } + body[field] = masked + } + return nil + } if user, ok := body["user"].(string); ok { masked, err := m.maskText("user", user) if err != nil { diff --git a/apps/gateway/internal/enclave/profile/manifest.go b/apps/gateway/internal/enclave/profile/manifest.go index dfc0be4..c7aa101 100644 --- a/apps/gateway/internal/enclave/profile/manifest.go +++ b/apps/gateway/internal/enclave/profile/manifest.go @@ -33,6 +33,7 @@ type ProviderEgressRoute struct { // production provider set. Keep this list in sync with the parent proxy units. func ProviderEgressRoutes() []ProviderEgressRoute { return []ProviderEgressRoute{ + {Name: "minimax", Host: "api.minimax.io", Port: 443, VsockPort: 8453}, {Name: "typesafe", Host: "api.typesafe.ai", Port: 443, VsockPort: 8452}, {Name: "deepseek", Host: "api.deepseek.com", Port: 443, VsockPort: 8443}, {Name: "novita", Host: "api.novita.ai", Port: 443, VsockPort: 8446}, diff --git a/apps/gateway/main.go b/apps/gateway/main.go index e1bc1a5..719cee6 100644 --- a/apps/gateway/main.go +++ b/apps/gateway/main.go @@ -99,6 +99,15 @@ func main() { log.Fatalf("invalid guest chat configuration: %v", err) } generateSvc.SetGuestMaxOutputTokens(guestMaxOutput) + imageTimeout, err := timeoutSeconds("IMAGE_GENERATION_TIMEOUT_SECONDS", services.DefaultImageTimeout) + if err != nil { + log.Fatalf("invalid image generation configuration: %v", err) + } + imageJobs, err := positiveIntEnv("IMAGE_GENERATION_CONCURRENCY_PER_ACCOUNT", services.DefaultImageConcurrency) + if err != nil { + log.Fatalf("invalid image generation configuration: %v", err) + } + generateSvc.SetImageOptions(imageTimeout, imageJobs) chatCompletionsSvc := services.NewChatCompletionsService(generateSvc, zapLogger) healthHandler := handlers.NewHealthHandler() @@ -141,7 +150,7 @@ func main() { } } - router := gwhttp.NewRouter(healthHandler, modelsHandler, chatHandler, handlers.NewSystemOneHandler(generateSvc, zapLogger), confidentialChatHandler, tinfoilHandler, attestationHandler, zapLogger) + router := gwhttp.NewRouter(healthHandler, modelsHandler, chatHandler, handlers.NewImageGenerationsHandler(generateSvc, zapLogger), handlers.NewSystemOneHandler(generateSvc, zapLogger), confidentialChatHandler, tinfoilHandler, attestationHandler, zapLogger) router = middleware.ReadTimeout(transportTimeouts.ClientIO)(middleware.WriteTimeout(transportTimeouts.ClientIO)(router)) metricsMux := http.NewServeMux() diff --git a/go.mod b/go.mod index b9bd0ac..6e1455b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/dappnode/dappnode-nexus-gateway -go 1.26.10 +go 1.26.11 require ( github.com/0xsequence/nsm v0.1.0 @@ -16,9 +16,10 @@ require ( github.com/tinfoilsh/encrypted-http-body-protocol v0.2.6 github.com/tinfoilsh/tinfoil-go v0.13.1 go.uber.org/zap v1.28.0 + golang.org/x/image v0.48.0 golang.org/x/net v0.60.0 - golang.org/x/sync v0.23.0 - golang.org/x/sys v0.48.0 + golang.org/x/sync v0.25.0 + golang.org/x/sys v0.50.0 ) require ( @@ -111,9 +112,9 @@ require ( go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.57.0 // indirect - golang.org/x/mod v0.41.0 // indirect + golang.org/x/mod v0.42.0 // indirect golang.org/x/term v0.46.0 // indirect - golang.org/x/text v0.42.0 // indirect + golang.org/x/text v0.44.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/grpc v1.83.2 // indirect diff --git a/go.sum b/go.sum index 9e37aad..ca97f0d 100644 --- a/go.sum +++ b/go.sum @@ -384,21 +384,23 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= -golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= -golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/image v0.48.0 h1:1AvuPcuoOTJpA7s4VZ5umrb90T3So0rkB/bSGmxkxik= +golang.org/x/image v0.48.0/go.mod h1:mmtg64iqJKGqEgy+r13dfybl85tmxUd2lMnmaiSdWYY= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= -golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs= +golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs= +golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= -golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= -golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo= +golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= diff --git a/pkg/domain/error.go b/pkg/domain/error.go index a8c8866..0fabd68 100644 --- a/pkg/domain/error.go +++ b/pkg/domain/error.go @@ -36,6 +36,10 @@ const ( ErrCodeInsufficientBalance = "insufficient_balance" ErrCodeDailyTokenLimit = "daily_token_limit_exceeded" ErrCodeGuestModelNotAllowed = "guest_model_not_allowed" + ErrCodeContentPolicy = "content_policy_violation" + ErrCodeTooManyConcurrent = "too_many_concurrent_requests" + ErrCodeIdempotencyInProgress = "idempotency_request_in_progress" + ErrCodeIdempotencyKeyReused = "idempotency_key_reused" ErrCodeConfidentialLimit = "confidential_request_limit_exceeded" ErrCodeClientCanceled = "client_canceled" ErrCodeInternalError = "internal_error" @@ -101,6 +105,33 @@ func ErrGuestModelNotAllowed(model string) *GatewayError { Message: fmt.Sprintf("Guest chat can't use '%s'. Sign up for free to use every model.", model)} } +// ErrContentPolicy is a prompt the provider refused to generate. Nothing is +// charged for it. +func ErrContentPolicy() *GatewayError { + return &GatewayError{HTTPStatus: 422, Type: ErrTypeInvalidRequest, Code: ErrCodeContentPolicy, + Message: "The provider refused this prompt under its content policy. Nothing was charged."} +} + +// ErrTooManyConcurrent limits how many slow jobs (image generations) one +// account runs at once. +func ErrTooManyConcurrent(limit int) *GatewayError { + return &GatewayError{HTTPStatus: 429, Type: ErrTypeRateLimit, Code: ErrCodeTooManyConcurrent, RetryAfter: "5", + Message: fmt.Sprintf("This account already has %d image generations running. Wait for one to finish.", limit)} +} + +// ErrIdempotencyInProgress is a retry that arrived while the first request +// with the same Idempotency-Key is still running. +func ErrIdempotencyInProgress() *GatewayError { + return &GatewayError{HTTPStatus: 409, Type: ErrTypeInvalidRequest, Code: ErrCodeIdempotencyInProgress, RetryAfter: "2", + Message: "A request with this Idempotency-Key is still running. Retry when it has finished."} +} + +// ErrIdempotencyKeyReused is a key already used for a finished request whose +// result the gateway no longer holds, or for a different request. +func ErrIdempotencyKeyReused(msg string) *GatewayError { + return &GatewayError{HTTPStatus: 409, Type: ErrTypeInvalidRequest, Code: ErrCodeIdempotencyKeyReused, Message: msg} +} + func ErrUnsupportedModel(model string) *GatewayError { return &GatewayError{HTTPStatus: 404, Type: ErrTypeInvalidRequest, Code: ErrCodeUnsupportedModel, Message: fmt.Sprintf("Model '%s' is not available", model)} } diff --git a/pkg/domain/error_class.go b/pkg/domain/error_class.go index fab1a3a..84f63cd 100644 --- a/pkg/domain/error_class.go +++ b/pkg/domain/error_class.go @@ -53,6 +53,10 @@ var faultByCode = map[string]string{ ErrCodeNotFound: FaultCaller, ErrCodeForbidden: FaultCaller, ErrCodeConflict: FaultCaller, + ErrCodeContentPolicy: FaultCaller, + ErrCodeTooManyConcurrent: FaultCaller, + ErrCodeIdempotencyInProgress: FaultCaller, + ErrCodeIdempotencyKeyReused: FaultCaller, ErrCodeAlreadyExists: FaultCaller, ErrCodeProviderRateLimited: FaultProvider, ErrCodeProviderUnavailable: FaultProvider, diff --git a/pkg/domain/generation.go b/pkg/domain/generation.go index 8a22524..742d97f 100644 --- a/pkg/domain/generation.go +++ b/pkg/domain/generation.go @@ -2,8 +2,9 @@ package domain // Endpoint constants. const ( - EndpointChatCompletions = "chat_completions" - EndpointSystemOne = "systemone" + EndpointChatCompletions = "chat_completions" + EndpointSystemOne = "systemone" + EndpointImageGenerations = "image_generations" ) // GenerateRequest is what the gateway reads from a client's request: enough @@ -36,6 +37,9 @@ type GenerateRequest struct { ParallelToolCalls *bool // StructuredOutput is set when the client asks for JSON output. StructuredOutput bool + // Images is how many images an image generation asks for. + Images *int + ImageEdit bool } // InputItem is one message: its role and text. diff --git a/pkg/domain/image_usage.go b/pkg/domain/image_usage.go new file mode 100644 index 0000000..f5bc837 --- /dev/null +++ b/pkg/domain/image_usage.go @@ -0,0 +1,33 @@ +package domain + +import "encoding/json" + +// ParseImageTokenUsage validates MiniMax's three separate billing dimensions. +// PromptTokens is text only; ImageInputTokens is never counted a second time. +func ParseImageTokenUsage(raw json.RawMessage, images int64) (Usage, bool) { + var v struct { + Input *int64 `json:"input_tokens"` + Output *int64 `json:"output_tokens"` + Total *int64 `json:"total_tokens"` + Images *int64 `json:"generated_images"` + InputDetails struct { + Text *int64 `json:"text_tokens"` + Image *int64 `json:"image_tokens"` + } `json:"input_tokens_details"` + OutputDetails struct { + Image *int64 `json:"image_tokens"` + } `json:"output_tokens_details"` + } + if json.Unmarshal(raw, &v) != nil { + return Usage{}, false + } + for _, count := range []*int64{v.Input, v.Output, v.Total, v.Images, v.InputDetails.Text, v.InputDetails.Image, v.OutputDetails.Image} { + if count == nil || *count < 0 || *count > 1_000_000_000 { + return Usage{}, false + } + } + if images <= 0 || *v.Images != images || *v.Input != *v.InputDetails.Text+*v.InputDetails.Image || *v.Output != *v.OutputDetails.Image || *v.Total != *v.Input+*v.Output || *v.Output == 0 { + return Usage{}, false + } + return Usage{PromptTokens: *v.InputDetails.Text, ImageInputTokens: *v.InputDetails.Image, CompletionTokens: *v.OutputDetails.Image, TotalTokens: *v.Total, Images: images, ProviderUsage: raw}, true +} diff --git a/pkg/domain/image_usage_test.go b/pkg/domain/image_usage_test.go new file mode 100644 index 0000000..20414f3 --- /dev/null +++ b/pkg/domain/image_usage_test.go @@ -0,0 +1,20 @@ +package domain + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestImageTokenUsageValidation(t *testing.T) { + raw := `{"input_tokens":3000,"output_tokens":4000,"total_tokens":7000,"generated_images":1,"input_tokens_details":{"text_tokens":1000,"image_tokens":2000},"output_tokens_details":{"image_tokens":4000}}` + usage, ok := ParseImageTokenUsage(json.RawMessage(raw), 1) + if !ok || usage.PromptTokens != 1000 || usage.ImageInputTokens != 2000 || usage.CompletionTokens != 4000 || usage.TotalTokens != 7000 { + t.Fatalf("usage=%+v valid=%v", usage, ok) + } + for _, invalid := range []string{`{}`, `null`, strings.Replace(raw, `"text_tokens":1000`, `"text_tokens":-1`, 1), strings.Replace(raw, `"input_tokens":3000`, `"input_tokens":2000`, 1), strings.Replace(raw, `"generated_images":1`, `"generated_images":2`, 1), strings.Replace(raw, `"image_tokens":2000`, `"image_tokens":null`, 1)} { + if _, ok := ParseImageTokenUsage(json.RawMessage(invalid), 1); ok { + t.Fatalf("accepted invalid usage: %s", invalid) + } + } +} diff --git a/pkg/domain/model.go b/pkg/domain/model.go index fd933ae..6cc17f4 100644 --- a/pkg/domain/model.go +++ b/pkg/domain/model.go @@ -2,6 +2,7 @@ package domain import ( "encoding/json" + "strings" "github.com/shopspring/decimal" ) @@ -41,6 +42,8 @@ type PublicModel struct { ProviderConfig ProviderConfig Fallback *ProviderTarget SupportsChatCompletions bool // Routing: whether chat is served. + // SupportsImageGenerations marks a model serving image requests. + SupportsImageGenerations bool // Capabilities is the operator's description of the model, a JSON object // published unchanged in /v1/models. Gateway never rejects requests by it. Capabilities json.RawMessage @@ -51,6 +54,10 @@ type PublicModel struct { OutputPricePerMillion decimal.Decimal CacheReadPricePerMillion *decimal.Decimal CacheWritePricePerMillion *decimal.Decimal + // Image token rates use InputPricePerMillion for text and OutputPricePerMillion + // for generated images. + ImageInputPricePerMillion *decimal.Decimal + ImageMaxCostPerImage *decimal.Decimal Currency string Active bool // GuestAllowed marks a model guest chat keys may call; they can call no @@ -79,7 +86,15 @@ func (m PublicModel) SupportsEndpoint(endpoint string) bool { switch endpoint { case EndpointChatCompletions: return m.SupportsChatCompletions + case EndpointImageGenerations: + return m.SupportsImageGenerations && m.SupportsImageEdits() && m.ImageInputPricePerMillion != nil && m.ImageMaxCostPerImage != nil default: return false } } + +// SupportsImageEdits is restricted to the implemented MiniMax editing protocol. +func (m PublicModel) SupportsImageEdits() bool { + return m.SupportsImageGenerations && strings.EqualFold(m.ProviderConfig.ProviderName, "minimax") && + (m.UpstreamModelName == "MiniMax-H3-Image" || m.UpstreamModelName == "MiniMax-H3-Image-Highspeed") +} diff --git a/pkg/domain/usage.go b/pkg/domain/usage.go index 6033e51..d8a8d15 100644 --- a/pkg/domain/usage.go +++ b/pkg/domain/usage.go @@ -1,10 +1,16 @@ package domain +import "encoding/json" + // Usage holds token usage information for a generation. type Usage struct { - PromptTokens int64 `json:"prompt_tokens"` - CompletionTokens int64 `json:"completion_tokens"` - TotalTokens int64 `json:"total_tokens"` - CacheCreationTokens int64 `json:"cache_creation_tokens"` - CacheReadTokens int64 `json:"cache_read_tokens"` + ProviderUsage json.RawMessage `json:"provider_usage,omitempty"` + ImageInputTokens int64 `json:"image_input_tokens,omitempty"` + PromptTokens int64 `json:"prompt_tokens"` + CompletionTokens int64 `json:"completion_tokens"` + TotalTokens int64 `json:"total_tokens"` + CacheCreationTokens int64 `json:"cache_creation_tokens"` + CacheReadTokens int64 `json:"cache_read_tokens"` + // Images is how many images an image generation returned. + Images int64 `json:"images,omitempty"` }