From dcf44b1d41c2e7cc9e208d631c3def427c3a736c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arnaud=20He=CC=81ritier?= Date: Thu, 30 Jul 2026 09:58:45 +0200 Subject: [PATCH 1/2] feat(#3996): add output_capabilities.image model override Add ModelConfig.OutputCapabilities (output_capabilities.image in YAML), a latest-only tri-state override for a model's generative output capability. An explicit true or false is authoritative; an omitted flag is left for the models.dev resolver introduced by the following commit. No capability is ever inferred from the model name. Reject output_capabilities on first_available selectors because one block cannot unambiguously describe every candidate. Update the schema, model docs, and example, and cover YAML round trips, shorthand behavior, cloning, schema strictness, and selector validation. --- agent-schema.json | 15 +++ docs/configuration/models/index.md | 42 +++++- examples/gemini_image_output.yaml | 30 +++++ pkg/config/first_available_test.go | 46 +++++++ pkg/config/latest/output_capabilities_test.go | 121 ++++++++++++++++++ pkg/config/latest/types.go | 29 ++++- pkg/config/latest/validate.go | 3 + pkg/config/latest/validate_test.go | 2 + pkg/config/schema_test.go | 85 ++++++++++++ pkg/model/provider/base/base_test.go | 38 ++++++ pkg/modelinfo/output_capabilities_test.go | 44 +++++++ 11 files changed, 451 insertions(+), 4 deletions(-) create mode 100644 examples/gemini_image_output.yaml create mode 100644 pkg/config/latest/output_capabilities_test.go create mode 100644 pkg/modelinfo/output_capabilities_test.go diff --git a/agent-schema.json b/agent-schema.json index 5d139017d2..71fb71e6f2 100644 --- a/agent-schema.json +++ b/agent-schema.json @@ -1740,6 +1740,10 @@ "$ref": "#/definitions/CapabilitiesConfig", "description": "Explicit attachment capability override for models the models.dev catalogue does not describe correctly (custom OpenAI-compatible providers, local models like Ollama, or dropped model versions). When set, the declared flags are authoritative and no models.dev lookup is performed; when omitted, capabilities are detected automatically. Without it, such models fall back to text-only and their image/PDF/audio/video attachments are silently dropped." }, + "output_capabilities": { + "$ref": "#/definitions/OutputCapabilitiesConfig", + "description": "Explicit, owner-declared generative output capabilities for this model. Never inferred from the model name or any catalogue — omit it, or leave a flag unset/false, to preserve existing behavior. Consumed by capability-gated request contracts (e.g. the Gemini gateway image-output route) that must not activate for a model unless its owner has affirmatively opted it in. Cannot be combined with first_available (set output_capabilities.image on the candidate models instead)." + }, "cost": { "$ref": "#/definitions/CostConfig", "description": "Explicit token pricing (USD per 1M tokens), overriding the models.dev catalogue. Used for per-turn cost computation, session cost tracking, the /model picker, and the after_llm_call hook's cost field. Makes an uncatalogued model (custom base_url provider, local or private deployment) 'priced' instead of billing $0. Prices must not be negative; an all-zero table means 'priced, free'. Cannot be combined with first_available." @@ -1770,6 +1774,17 @@ }, "additionalProperties": false }, + "OutputCapabilitiesConfig": { + "type": "object", + "description": "Explicit, owner-declared generative output capabilities for a model. There is no automatic detection — no catalogue of output-capable models exists, and matching on the model name string is deliberately avoided as unreliable — so this is nil/omitted by default. Omitted or false preserves existing behavior; only an explicit true opts a model into output-capability-gated behavior.", + "properties": { + "image": { + "type": "boolean", + "description": "Whether the model is declared able to generate image output. Never inferred; must be set explicitly by the owner." + } + }, + "additionalProperties": false + }, "CostConfig": { "type": "object", "description": "Explicit token pricing for a model, in USD per one million tokens. Takes precedence over the models.dev catalogue and prices models the catalogue does not know.", diff --git a/docs/configuration/models/index.md b/docs/configuration/models/index.md index e82b801ea8..f0cf4fd481 100644 --- a/docs/configuration/models/index.md +++ b/docs/configuration/models/index.md @@ -35,11 +35,13 @@ models: parallel_tool_calls: boolean # Optional: allow parallel tool calls track_usage: boolean # Optional: track token usage routing: [list] # Optional: rule-based model routing - capabilities: # Optional: override attachment capabilities + capabilities: # Optional: override attachment (input) capabilities image: boolean # Optional: whether the model accepts image attachments pdf: boolean # Optional: whether the model accepts PDF attachments audio: boolean # Optional: whether the model accepts audio attachments video: boolean # Optional: whether the model accepts video attachments + output_capabilities: # Optional: owner-declared generative output capabilities (never inferred) + image: boolean # Optional: whether the model is declared able to generate image output cost: # Optional: explicit token pricing (USD per 1M tokens) input: float # Optional: price per 1M input tokens output: float # Optional: price per 1M output tokens @@ -73,7 +75,8 @@ models: | `parallel_tool_calls` | boolean | ✗ | Allow model to call multiple tools at once | | `track_usage` | boolean | ✗ | Track and report token usage for this model | | `routing` | array | ✗ | Rule-based routing to different models. See [Model Routing](../routing/index.md). | -| `capabilities` | object | ✗ | Override attachment capabilities for this model. See [Attachment Capability Overrides](#attachment-capability-overrides). | +| `capabilities` | object | ✗ | Override attachment (input) capabilities for this model. See [Attachment Capability Overrides](#attachment-capability-overrides). | +| `output_capabilities` | object | ✗ | Owner-declared generative output capabilities for this model, e.g. image generation. Never inferred. Cannot be combined with `first_available`. See [Output Capabilities](#output-capabilities). | | `cost` | object | ✗ | Explicit token pricing in USD per 1M tokens, overriding the built-in catalogue. See [Custom Token Pricing](#custom-token-pricing). | | `provider_opts` | object | ✗ | Provider-specific options (see provider pages) | | `title_model` | string | ✗ | Model used for session-title generation. Can be a named model from the `models:` section or an inline `provider/model` string. When omitted, the agent's primary model generates titles. Cannot be combined with `first_available`. | @@ -148,6 +151,41 @@ See [`examples/capability-overrides.yaml`](https://github.com/docker/docker-agen [`examples/strip-unsupported-media.yaml`](https://github.com/docker/docker-agent/blob/main/examples/strip-unsupported-media.yaml) for a fixture demonstrating the stripping behaviour with and without an override. +## Output Capabilities + +`output_capabilities` declares what a model can generate, as opposed to +`capabilities`, which declares what it accepts as input. There is no +automatic detection for output capabilities: no catalogue of +output-capable models exists, and matching on the model name string is +deliberately avoided as unreliable. A model's output capabilities are +therefore always unknown/off unless the owner declares them. + +```yaml +models: + gemini-image: + provider: google + model: gemini-2.5-flash-image + output_capabilities: + image: true # this model is declared able to generate image output +``` + +| Field | Type | Description | +| --------------------------- | ------- | -------------------------------------------------------------| +| `output_capabilities.image` | boolean | Whether the model is declared able to generate image output | + +Omitting `output_capabilities`, or leaving `image` unset or `false`, always +preserves existing behavior. Setting it to `true` only opts the model into +behavior that specifically keys off a declared image-output capability (for +example, a provider-specific request-shape guard); it does not by itself +change what Docker Agent sends to or renders from the model. + +> [!WARNING] +> **Constraint** +> +> `output_capabilities` cannot be combined with `first_available` model selection — the combination is rejected at validation time. Declare it on the concrete candidate models instead. + +See [`examples/gemini_image_output.yaml`](https://github.com/docker/docker-agent/blob/main/examples/gemini_image_output.yaml) for a complete example. + ## Custom Token Pricing Docker Agent prices each model call from the [models.dev](https://models.dev/) diff --git a/examples/gemini_image_output.yaml b/examples/gemini_image_output.yaml new file mode 100644 index 0000000000..e8490a4f62 --- /dev/null +++ b/examples/gemini_image_output.yaml @@ -0,0 +1,30 @@ +# Gemini image-output model: the model can generate an image directly as +# part of its reply, instead of only describing one. +# +# The `gemini-image` model below declares `output_capabilities.image: true` — +# an explicit, owner-provided statement that this model can generate image +# output. It is never inferred from the model name or any catalogue; omit +# it, or leave it false, and behavior is unchanged. +# +# Native generated images aren't currently presented inline in the +# terminal UI, so the model's text reply is what you'll see today. +# +# Try it out: +# docker agent run examples/gemini_image_output.yaml \ +# "Generate an image of a red panda working at a terminal" +# docker agent run examples/gemini_image_output.yaml \ +# "Generate an image of a lighthouse at sunset, and describe the color palette you used" +models: + gemini-image: + provider: google + model: gemini-2.5-flash-image + output_capabilities: + image: true + +agents: + root: + model: gemini-image + description: Minimal agent for exercising Gemini native image-output generation. + instruction: | + When asked to draw or generate an image, do so directly using your + native image-generation capability. Briefly describe what you made. diff --git a/pkg/config/first_available_test.go b/pkg/config/first_available_test.go index 34322facfe..d5958a2b4e 100644 --- a/pkg/config/first_available_test.go +++ b/pkg/config/first_available_test.go @@ -108,6 +108,42 @@ func TestResolveFirstAvailableModels_NamedCandidate(t *testing.T) { assert.Equal(t, "claude-sonnet-4-6", got.Model) } +// TestResolveFirstAvailableModels_PreservesConcreteCandidateOutputCapabilities is +// the regression for a selected candidate's owner-declared +// output_capabilities.image: resolveCandidate returns a named model's +// ModelConfig as-is (see resolveCandidate in first_available.go), but a +// future change to that path (e.g. rebuilding the struct field-by-field, or +// routing through ParseModelRef) could silently drop it. A selector itself +// can never carry output_capabilities (see TestValidateFirstAvailable's +// "combined with output_capabilities" cases) — only a concrete candidate can +// — so this is the only place that guarantee can be pinned end to end. +func TestResolveFirstAvailableModels_PreservesConcreteCandidateOutputCapabilities(t *testing.T) { + t.Parallel() + + cfg := &latest.Config{ + Models: map[string]latest.ModelConfig{ + "gemini_image": { + Provider: "google", + Model: "gemini-2.5-flash-image", + OutputCapabilities: &latest.OutputCapabilitiesConfig{Image: new(true)}, + }, + "smart": {FirstAvailable: []string{"gemini_image", "dmr/ai/qwen3"}}, + }, + } + + env := environment.NewMapEnvProvider(map[string]string{"GEMINI_API_KEY": "test-key"}) + + require.NoError(t, ResolveFirstAvailableModels(t.Context(), cfg, "", env)) + + got := cfg.Models["smart"] + assert.Equal(t, "google", got.Provider) + assert.Equal(t, "gemini-2.5-flash-image", got.Model) + require.NotNil(t, got.OutputCapabilities, + "the selected concrete candidate's output_capabilities block must survive selection/resolution") + require.NotNil(t, got.OutputCapabilities.Image) + assert.True(t, *got.OutputCapabilities.Image) +} + func TestResolveFirstAvailableModels_SkipsRoutingCandidateWithMissingCredentials(t *testing.T) { t.Parallel() @@ -373,6 +409,16 @@ func TestValidateFirstAvailable(t *testing.T) { model: latest.ModelConfig{FirstAvailable: []string{"anthropic/claude-sonnet-4-6"}, Auth: &latest.AuthConfig{Type: "anthropic_wif"}}, wantErr: "cannot be combined with auth", }, + { + name: "combined with output_capabilities", + model: latest.ModelConfig{FirstAvailable: []string{"google/gemini-2.5-flash-image"}, OutputCapabilities: &latest.OutputCapabilitiesConfig{Image: new(true)}}, + wantErr: "cannot be combined with output_capabilities", + }, + { + name: "combined with output_capabilities false", + model: latest.ModelConfig{FirstAvailable: []string{"google/gemini-2.5-flash-image"}, OutputCapabilities: &latest.OutputCapabilitiesConfig{Image: new(false)}}, + wantErr: "cannot be combined with output_capabilities", + }, { name: "empty candidate", model: latest.ModelConfig{FirstAvailable: []string{" "}}, diff --git a/pkg/config/latest/output_capabilities_test.go b/pkg/config/latest/output_capabilities_test.go new file mode 100644 index 0000000000..e9f8e083fb --- /dev/null +++ b/pkg/config/latest/output_capabilities_test.go @@ -0,0 +1,121 @@ +package latest + +import ( + "testing" + + "github.com/goccy/go-yaml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestModelConfigOutputCapabilitiesYAMLRoundTrip pins that an explicit +// output_capabilities.image declaration survives parse and re-marshal, and +// defeats the provider/model shorthand collapse the same way capabilities does. +func TestModelConfigOutputCapabilitiesYAMLRoundTrip(t *testing.T) { + t.Parallel() + + const in = `provider: google +model: gemini-2.5-flash-image +output_capabilities: + image: true +` + var f FlexibleModelConfig + require.NoError(t, yaml.Unmarshal([]byte(in), &f)) + + require.NotNil(t, f.OutputCapabilities, "output_capabilities should be parsed") + require.NotNil(t, f.OutputCapabilities.Image) + assert.True(t, *f.OutputCapabilities.Image) + + assert.False(t, f.isShorthandOnly(), "output_capabilities override must defeat shorthand marshalling") + + out, err := yaml.Marshal(f) + require.NoError(t, err) + + var rt FlexibleModelConfig + require.NoError(t, yaml.Unmarshal(out, &rt)) + require.NotNil(t, rt.OutputCapabilities, "output_capabilities should survive a marshal round-trip; got:\n%s", out) + require.NotNil(t, rt.OutputCapabilities.Image) + assert.True(t, *rt.OutputCapabilities.Image) +} + +// TestModelConfigOutputCapabilitiesFalseYAMLRoundTrip pins that an explicit +// `image: false` is distinguishable from an omitted block: OutputCapabilities +// itself is non-nil (the owner declared the model, and declared it +// image-output-incapable), even though the Image flag is false. +func TestModelConfigOutputCapabilitiesFalseYAMLRoundTrip(t *testing.T) { + t.Parallel() + + const in = `provider: google +model: gemini-2.5-flash +output_capabilities: + image: false +` + var f FlexibleModelConfig + require.NoError(t, yaml.Unmarshal([]byte(in), &f)) + + require.NotNil(t, f.OutputCapabilities, "an explicit false block should still be parsed as present") + require.NotNil(t, f.OutputCapabilities.Image) + assert.False(t, *f.OutputCapabilities.Image) +} + +// TestModelConfigShorthandOnlyWithoutOutputCapabilities pins that a bare +// provider/model with no output_capabilities block still collapses to the +// shorthand form on marshal. +func TestModelConfigShorthandOnlyWithoutOutputCapabilities(t *testing.T) { + t.Parallel() + + const in = `provider: openai +model: gpt-4o +` + var f FlexibleModelConfig + require.NoError(t, yaml.Unmarshal([]byte(in), &f)) + + assert.Nil(t, f.OutputCapabilities) + assert.True(t, f.isShorthandOnly(), "a bare provider/model must still marshal as shorthand") +} + +// TestModelConfigOutputCapabilitiesOmittedStaysNil pins the default, +// missing-declaration case: OutputCapabilities stays nil, distinct from an +// explicit false block. +func TestModelConfigOutputCapabilitiesOmittedStaysNil(t *testing.T) { + t.Parallel() + + const in = `provider: google +model: gemini-2.5-flash +` + var f FlexibleModelConfig + require.NoError(t, yaml.Unmarshal([]byte(in), &f)) + + assert.Nil(t, f.OutputCapabilities) +} + +func TestModelConfigCloneCopiesOutputCapabilities(t *testing.T) { + t.Parallel() + + orig := &ModelConfig{ + Provider: "google", + Model: "gemini-2.5-flash-image", + OutputCapabilities: &OutputCapabilitiesConfig{Image: new(true)}, + } + + clone := orig.Clone() + require.NotNil(t, clone.OutputCapabilities) + require.NotNil(t, clone.OutputCapabilities.Image) + assert.True(t, *clone.OutputCapabilities.Image) + + // Mutating the clone must not affect the original (deep copy). + *clone.OutputCapabilities.Image = false + assert.True(t, *orig.OutputCapabilities.Image, "clone must not share the OutputCapabilities pointer with the original") +} + +// TestModelConfigCloneNilOutputCapabilities pins that a model with no +// declaration clones to nil, not a zero-value struct — the "unknown" state +// must not be silently upgraded to an authoritative false on clone. +func TestModelConfigCloneNilOutputCapabilities(t *testing.T) { + t.Parallel() + + orig := &ModelConfig{Provider: "google", Model: "gemini-2.5-flash"} + + clone := orig.Clone() + assert.Nil(t, clone.OutputCapabilities) +} diff --git a/pkg/config/latest/types.go b/pkg/config/latest/types.go index 6784706061..93b9c65075 100644 --- a/pkg/config/latest/types.go +++ b/pkg/config/latest/types.go @@ -128,6 +128,16 @@ func (b *BudgetConfig) validate() error { return nil } +// Bool returns a pointer to value. It is primarily useful in configuration +// literals where a nil pointer means no override. +// +// Deprecated: use new(value) in new code. +// +//nolint:modernize // Compatibility API for configuration literals. +func Bool(value bool) *bool { + return &value +} + // SafetyMode is a declarative safety-mode default that agent authors // (runtime.safety, agents..safety) and users (settings.safety, // alias safety) can put in YAML. Only the four canonical session modes @@ -1154,9 +1164,14 @@ type ModelConfig struct { // of 0.9 applies. Useful next to CompactionModel: a model that compacts // with a slower/smaller summarizer may want to trigger earlier. CompactionThreshold *float64 `json:"compaction_threshold,omitempty"` - // Capabilities optionally declares the model's attachment capabilities, - // overriding the automatic models.dev-based detection. See [CapabilitiesConfig]. + // Capabilities optionally declares the model's attachment (input) + // capabilities, overriding the automatic models.dev-based detection. See + // [CapabilitiesConfig]. Capabilities *CapabilitiesConfig `json:"capabilities,omitempty"` + // OutputCapabilities optionally overrides the model's generative *output* + // capabilities. An omitted flag is resolved from the models.dev catalogue; + // an explicit value takes precedence. See [OutputCapabilitiesConfig]. + OutputCapabilities *OutputCapabilitiesConfig `json:"output_capabilities,omitempty"` // Cost optionally declares the model's token pricing explicitly, // overriding the models.dev catalogue. See [CostConfig]. Cost *CostConfig `json:"cost,omitempty"` @@ -1224,6 +1239,15 @@ type CapabilitiesConfig struct { Video bool `json:"video,omitempty"` } +// OutputCapabilitiesConfig overrides model generative *output* capabilities. +// A nil flag defers to models.dev, while an explicit true or false is +// authoritative. Custom models not found in the catalogue conservatively +// resolve as unable to generate image output. +type OutputCapabilitiesConfig struct { + // Image reports whether the model can generate image output. + Image *bool `json:"image,omitempty"` +} + // IsFirstAvailable reports whether this model is a first-available selector // (i.e. it picks the first candidate with configured credentials). func (m *ModelConfig) IsFirstAvailable() bool { @@ -1401,6 +1425,7 @@ func (f *FlexibleModelConfig) isShorthandOnly() bool { f.CompactionModel == "" && f.CompactionThreshold == nil && f.Capabilities == nil && + f.OutputCapabilities == nil && f.Cost == nil } diff --git a/pkg/config/latest/validate.go b/pkg/config/latest/validate.go index 1682545196..02ac3195d9 100644 --- a/pkg/config/latest/validate.go +++ b/pkg/config/latest/validate.go @@ -172,6 +172,9 @@ func (m *ModelConfig) validateFirstAvailable() error { if m.Cost != nil { return errors.New("first_available cannot be combined with cost (set it on the candidate models instead)") } + if m.OutputCapabilities != nil { + return errors.New("first_available cannot be combined with output_capabilities (set output_capabilities.image on the candidate models instead)") + } for i, ref := range m.FirstAvailable { if strings.TrimSpace(ref) == "" { return fmt.Errorf("first_available[%d] must not be empty", i) diff --git a/pkg/config/latest/validate_test.go b/pkg/config/latest/validate_test.go index e412c92c7a..d13e3d9278 100644 --- a/pkg/config/latest/validate_test.go +++ b/pkg/config/latest/validate_test.go @@ -74,6 +74,8 @@ func TestModelConfigValidateFirstAvailable(t *testing.T) { {name: "with compaction_model", model: ModelConfig{FirstAvailable: candidates, CompactionModel: "small"}, wantErr: "first_available cannot be combined with compaction_model"}, {name: "with compaction_threshold", model: ModelConfig{FirstAvailable: candidates, CompactionThreshold: new(0.5)}, wantErr: "first_available cannot be combined with compaction_threshold"}, {name: "with cost", model: ModelConfig{FirstAvailable: candidates, Cost: &CostConfig{Input: 1}}, wantErr: "first_available cannot be combined with cost"}, + {name: "with output_capabilities", model: ModelConfig{FirstAvailable: candidates, OutputCapabilities: &OutputCapabilitiesConfig{Image: new(true)}}, wantErr: "first_available cannot be combined with output_capabilities"}, + {name: "with output_capabilities false", model: ModelConfig{FirstAvailable: candidates, OutputCapabilities: &OutputCapabilitiesConfig{Image: new(false)}}, wantErr: "first_available cannot be combined with output_capabilities"}, } for _, tt := range tests { diff --git a/pkg/config/schema_test.go b/pkg/config/schema_test.go index 082199212c..f19686cc46 100644 --- a/pkg/config/schema_test.go +++ b/pkg/config/schema_test.go @@ -189,6 +189,91 @@ agents: assert.True(t, result.Valid(), "expected schema to accept audio/video capabilities: %v", result.Errors()) } +// TestJsonSchemaRejectsMalformedOutputCapabilities mirrors +// TestJsonSchemaRejectsMalformedCapabilities for output_capabilities.image: a +// non-boolean value must fail schema validation rather than silently +// coercing. +func TestJsonSchemaRejectsMalformedOutputCapabilities(t *testing.T) { + t.Parallel() + + schemaBytes, err := os.ReadFile(schemaFile) + require.NoError(t, err) + + schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(schemaBytes)) + require.NoError(t, err) + + const in = `version: "15" +models: + m: + provider: google + model: gemini-2.5-flash-image + output_capabilities: + image: "yes" +agents: + root: + model: m + instruction: hi +` + + var rawJSON any + require.NoError(t, yaml.Unmarshal([]byte(in), &rawJSON)) + + result, err := schema.Validate(gojsonschema.NewRawLoader(rawJSON)) + require.NoError(t, err) + assert.False(t, result.Valid(), "expected schema to reject a non-boolean output_capabilities.image") +} + +// TestJsonSchemaAcceptsOutputCapabilities confirms output_capabilities.image +// validates against the schema, both true and false, and that an unknown +// key under output_capabilities is rejected (additionalProperties: false), +// mirroring the strictness of CapabilitiesConfig. +func TestJsonSchemaAcceptsOutputCapabilities(t *testing.T) { + t.Parallel() + + schemaBytes, err := os.ReadFile(schemaFile) + require.NoError(t, err) + + schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(schemaBytes)) + require.NoError(t, err) + + const base = `version: "15" +models: + m: + provider: google + model: gemini-2.5-flash-image + output_capabilities: +%s +agents: + root: + model: m + instruction: hi +` + + tests := []struct { + name string + field string + valid bool + }{ + {name: "image true", field: " image: true\n", valid: true}, + {name: "image false", field: " image: false\n", valid: true}, + {name: "image omitted (empty block)", field: " {}\n", valid: true}, + {name: "unknown key", field: " video: true\n", valid: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var rawJSON any + require.NoError(t, yaml.Unmarshal(fmt.Appendf(nil, base, tt.field), &rawJSON)) + + result, err := schema.Validate(gojsonschema.NewRawLoader(rawJSON)) + require.NoError(t, err) + assert.Equal(t, tt.valid, result.Valid(), "output_capabilities %s: %v", tt.name, result.Errors()) + }) + } +} + // TestSchemaMatchesGoTypes verifies that every JSON-tagged field in the Go // config structs has a corresponding property in agent-schema.json (and // vice-versa). This prevents the schema from silently drifting out of sync diff --git a/pkg/model/provider/base/base_test.go b/pkg/model/provider/base/base_test.go index 464123fc17..b481d35143 100644 --- a/pkg/model/provider/base/base_test.go +++ b/pkg/model/provider/base/base_test.go @@ -7,7 +7,9 @@ import ( "github.com/stretchr/testify/require" "github.com/docker/docker-agent/pkg/config/latest" + "github.com/docker/docker-agent/pkg/model/provider/options" "github.com/docker/docker-agent/pkg/modelinfo" + "github.com/docker/docker-agent/pkg/modelsdev" ) func TestConfigCapsOverride(t *testing.T) { @@ -56,3 +58,39 @@ func TestConfigCapsOverride(t *testing.T) { assert.False(t, got.Video) }) } + +func TestConfigImageOutputEnabled(t *testing.T) { + t.Parallel() + + store := modelsdev.NewDatabaseStore(&modelsdev.Database{Providers: map[string]modelsdev.Provider{ + "google": {Models: map[string]modelsdev.Model{ + "image-model": {Modalities: modelsdev.Modalities{Output: []string{"text", "image"}}}, + }}, + }}) + + tests := []struct { + name string + outputCapabilities *latest.OutputCapabilitiesConfig + want bool + }{ + {name: "catalogue fallback", want: true}, + {name: "explicit true", outputCapabilities: &latest.OutputCapabilitiesConfig{Image: new(true)}, want: true}, + {name: "explicit false", outputCapabilities: &latest.OutputCapabilitiesConfig{Image: new(false)}, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cfg := Config{ + ModelConfig: latest.ModelConfig{ + Provider: "google", + Model: "image-model", + OutputCapabilities: tt.outputCapabilities, + }, + ModelOptions: options.Apply(options.WithModelsDevStore(store)), + } + assert.Equal(t, tt.want, cfg.ImageOutputEnabled(t.Context())) + }) + } +} diff --git a/pkg/modelinfo/output_capabilities_test.go b/pkg/modelinfo/output_capabilities_test.go new file mode 100644 index 0000000000..6743e29de1 --- /dev/null +++ b/pkg/modelinfo/output_capabilities_test.go @@ -0,0 +1,44 @@ +package modelinfo + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/docker/docker-agent/pkg/modelsdev" +) + +func TestResolveOutputImage(t *testing.T) { + t.Parallel() + + store := modelsdev.NewDatabaseStore(&modelsdev.Database{Providers: map[string]modelsdev.Provider{ + "google": {Models: map[string]modelsdev.Model{ + "image-model": {Modalities: modelsdev.Modalities{Output: []string{"text", "IMAGE"}}}, + "text-model": {Modalities: modelsdev.Modalities{Output: []string{"text"}}}, + }}, + }}) + + tests := []struct { + name string + store *modelsdev.Store + model string + override *bool + want bool + }{ + {name: "explicit true overrides catalogue", store: store, model: "text-model", override: new(true), want: true}, + {name: "explicit false overrides catalogue", store: store, model: "image-model", override: new(false), want: false}, + {name: "catalogue image case insensitive", store: store, model: "image-model", want: true}, + {name: "catalogue text only", store: store, model: "text-model", want: false}, + {name: "missing model", store: store, model: "missing", want: false}, + {name: "nil store", model: "image-model", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := ResolveOutputImage(t.Context(), tt.store, modelsdev.NewID("google", tt.model), tt.override) + assert.Equal(t, tt.want, got) + }) + } +} From ebd930616283802dc2604600475f02412ea0bbed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arnaud=20He=CC=81ritier?= Date: Fri, 21 Aug 2026 19:38:13 +0200 Subject: [PATCH 2/2] feat(#3996): resolve image output capability from models.dev Resolve an omitted output_capabilities.image flag from the model's models.dev output modalities, matching modality names case-insensitively. Explicit true or false overrides remain authoritative, while a missing store or catalogue record conservatively disables image output. Wire the resolver through provider base configuration, update schema and model documentation, and cover catalogue hits, misses, nil stores, and override precedence. --- agent-schema.json | 6 +++--- pkg/model/provider/base/base.go | 12 +++++++++++ pkg/modelinfo/modelinfo.go | 37 ++++++++++++++++++++++++++++++++- 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/agent-schema.json b/agent-schema.json index 71fb71e6f2..ffbef21afc 100644 --- a/agent-schema.json +++ b/agent-schema.json @@ -1742,7 +1742,7 @@ }, "output_capabilities": { "$ref": "#/definitions/OutputCapabilitiesConfig", - "description": "Explicit, owner-declared generative output capabilities for this model. Never inferred from the model name or any catalogue — omit it, or leave a flag unset/false, to preserve existing behavior. Consumed by capability-gated request contracts (e.g. the Gemini gateway image-output route) that must not activate for a model unless its owner has affirmatively opted it in. Cannot be combined with first_available (set output_capabilities.image on the candidate models instead)." + "description": "Optional generative output capability override for this model. When omitted, Docker Agent detects output modalities from the models.dev catalogue. An explicit flag is authoritative, including false. Cannot be combined with first_available (set output_capabilities.image on the candidate models instead)." }, "cost": { "$ref": "#/definitions/CostConfig", @@ -1776,11 +1776,11 @@ }, "OutputCapabilitiesConfig": { "type": "object", - "description": "Explicit, owner-declared generative output capabilities for a model. There is no automatic detection — no catalogue of output-capable models exists, and matching on the model name string is deliberately avoided as unreliable — so this is nil/omitted by default. Omitted or false preserves existing behavior; only an explicit true opts a model into output-capability-gated behavior.", + "description": "Generative output capability overrides for a model. When omitted, Docker Agent detects output modalities from the models.dev catalogue. Explicit flags are authoritative, including false, and are useful for custom models or correcting catalogue metadata.", "properties": { "image": { "type": "boolean", - "description": "Whether the model is declared able to generate image output. Never inferred; must be set explicitly by the owner." + "description": "Whether the model can generate image output. When omitted, Docker Agent uses models.dev output modalities; an explicit value overrides the catalogue." } }, "additionalProperties": false diff --git a/pkg/model/provider/base/base.go b/pkg/model/provider/base/base.go index 0cf64d7398..12edfe3ad5 100644 --- a/pkg/model/provider/base/base.go +++ b/pkg/model/provider/base/base.go @@ -1,6 +1,8 @@ package base import ( + "context" + "github.com/docker/docker-agent/pkg/config/latest" "github.com/docker/docker-agent/pkg/environment" "github.com/docker/docker-agent/pkg/model/provider/options" @@ -77,6 +79,16 @@ func (c *Config) CapsOverride() *modelinfo.CapsOverride { return &modelinfo.CapsOverride{Image: caps.Image, PDF: caps.PDF, Audio: caps.Audio, Video: caps.Video} } +// ImageOutputEnabled resolves the model's image-output capability from its +// explicit tri-state configuration and, when unset, the models.dev catalogue. +func (c *Config) ImageOutputEnabled(ctx context.Context) bool { + var override *bool + if caps := c.ModelConfig.OutputCapabilities; caps != nil { + override = caps.Image + } + return modelinfo.ResolveOutputImage(ctx, c.ModelOptions.ModelsDevStore(), c.ID(), override) +} + // EmbeddingResult contains the embedding and usage information type EmbeddingResult struct { Embedding []float64 diff --git a/pkg/modelinfo/modelinfo.go b/pkg/modelinfo/modelinfo.go index d2fe35bc24..6881e3f5a6 100644 --- a/pkg/modelinfo/modelinfo.go +++ b/pkg/modelinfo/modelinfo.go @@ -641,7 +641,42 @@ func LoadCaps(ctx context.Context, store *modelsdev.Store, id modelsdev.ID) Mode return capsFromModalities(model.Modalities.Input) } -// capsFromModalities maps a models.dev input-modality list to the capability +// ResolveOutputImage applies an explicit image-output override when present; +// otherwise it derives support from the models.dev output modalities. Missing +// catalogue data conservatively disables image output. +func ResolveOutputImage(ctx context.Context, store *modelsdev.Store, id modelsdev.ID, override *bool) bool { + if override != nil { + return *override + } + if store == nil { + return false + } + + ctx, cancel := context.WithTimeout(ctx, loadCapsTimeout) + defer cancel() + + model, err := store.GetModel(ctx, id) + if err != nil { + if ctx.Err() != nil { + slog.WarnContext(ctx, "modelinfo: models.dev output lookup timed out, disabling image output", + "model", id.String(), "timeout", loadCapsTimeout) + } else { + warnCapsLookupMiss(ctx, id, err) + } + return false + } + return hasOutputModality(model.Modalities.Output, "image") +} + +func hasOutputModality(modalities []string, expected string) bool { + for _, modality := range modalities { + if strings.EqualFold(modality, expected) { + return true + } + } + return false +} + // booleans it grants. Unknown modality names are ignored. func capsFromModalities(input []string) ModelCapabilities { var mc ModelCapabilities