diff --git a/.agents/skills/catalogue/SKILL.md b/.agents/skills/catalogue/SKILL.md index d5049ba..b5a5dfc 100644 --- a/.agents/skills/catalogue/SKILL.md +++ b/.agents/skills/catalogue/SKILL.md @@ -2,7 +2,7 @@ name: catalogue description: > Research, add, update, deprecate, or remove AI model catalogue entries — - text, image, video, audio, 3D, image tools, and embedding models — with + text, decision, image, video, audio, 3D, image tools, and embedding models — with their capabilities, limits, provider bindings, and published prices. Covers what an entry is, per-provider ID rules, regular pricing, native reference and provider coverage, public distribution, and the verification @@ -143,6 +143,16 @@ pricing shapes in use and their rules are in [`pricing.md`](./pricing.md). 5. **Record qualifications** that do not fit the JSON (regular-price evidence, long-context bands, ambiguous units) in `catalogue/PROVENANCE.md`, with source URLs. +## Decision offerings + +Decision endpoints are distinct offerings even when they share the underlying chat +model. Author them in `decision.models`, with `underlying_model` pointing at that +canonical model. Qualify each provider's exact `interface`, question vocabulary, +input envelope and decision-only rates. Do not copy chat cache/output prices, tools, +reasoning parameters or limits. Put contract differences on sourced +`decision_contract` records and unresolved conflicts in provenance. See +[`catalogue/README.md`](../../../catalogue/README.md#decision-offerings). + ## Reference pattern: text and image additions The GPT-6 Sol, GPT-6 Luna, GPT-6.1 Sol and Gemini Nano Banana 2.1 cards demonstrate diff --git a/catalogue/README.md b/catalogue/README.md index 86c8add..d9f9339 100644 --- a/catalogue/README.md +++ b/catalogue/README.md @@ -34,6 +34,7 @@ section is a table of cards keyed by card ID: | section | cards | | --------------------------------- | -------------------------------------------- | +| `decision.models` (optional) | Bounded classification and scoring offerings | | `text.catalog` | Language models | | `image.models` | Image generation, with per-provider bindings | | `video.models` | Video generation, with per-provider bindings | @@ -263,3 +264,31 @@ Single-provider cards and `three_d.rig_check.operation` may also carry `endpoints`, with the same shape as a binding's endpoint variants. New sourced objects have strict field vocabularies; pre-existing card and price shapes remain unchanged in schema version 1. + +## Decision offerings + +The optional `decision.models` section describes bounded decision offerings separately +from text generation. Existing version 1 snapshots remain valid without the section; +the generator emits it only when a card is authored there. A decision card uses the +common identity, vendor, release and provider fields, and requires `underlying_model` +(the canonical identity of the underlying model) and nonempty provider bindings. +This relationship does not inherit that model's chat capabilities, limits or prices. +Decision cards reject chat-only top-level fields such as `context_window`, `output_limit` +and `cost`; use sourced provider limits and prices for this offering's exact interface. + +Optional `decision_contract` on a card, binding or endpoint records sourced descriptive +facts: `question_types` (exact wire vocabulary, such as `predicate` or `noul`), +`input_forms` (`string`, `user_messages`, or `state`), `question_container` +(`array` or `named_object`), `image_input_forms` (`data_url`, `http_url`, or `file_id`), +`image_details`, `maximum_questions`, `maximum_images`, choice and score bounds +(`minimum_choices`, `maximum_choices`, `minimum_score_levels`, `maximum_score_levels`), +`typed_choice_values`, and `partial_refusals`. Every object requires a first-party +`source_url`; omitted fields are unrecorded, not unsupported. Minimum and maximum +pairs cannot be inverted. Typed choice values distinguish a boolean from its string +spelling; a false value records that the provider rejects such a collision. + +The endpoint's `interface` and source link remain authoritative for the full wire +contract and answer semantics. Provider vocabularies, image envelopes and pricing +remain independent even when the underlying model is identical. No ZDR eligibility, +probability calibration guarantee or output-generation pricing is inferred from +this family membership. diff --git a/catalogue/schemas/models.schema.json b/catalogue/schemas/models.schema.json index 53ec55d..779de8f 100644 --- a/catalogue/schemas/models.schema.json +++ b/catalogue/schemas/models.schema.json @@ -174,6 +174,13 @@ "$ref": "#/$defs/provider_coverage" } } + }, + "decision.models": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "$ref": "#/$defs/decision_card" + } } }, "$defs": { @@ -391,6 +398,9 @@ "type": "string", "format": "uri", "pattern": "^https://[^/]+(?:/|$)" + }, + "decision_contract": { + "$ref": "#/$defs/decision_contract" } } }, @@ -1101,6 +1111,106 @@ } } } + }, + "decision_contract": { + "$ref": "#/$defs/decision_contract" + } + } + }, + "decision_contract": { + "type": "object", + "additionalProperties": false, + "required": ["source_url"], + "minProperties": 2, + "properties": { + "source_url": { + "type": "string", + "format": "uri", + "pattern": "^https://[^/]+(?:/|$)" + }, + "question_types": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": ["predicate", "noul", "choice", "score"] + } + }, + "input_forms": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": ["string", "user_messages", "state"] + } + }, + "question_container": { + "enum": ["array", "named_object"] + }, + "image_input_forms": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": ["data_url", "http_url", "file_id"] + } + }, + "image_details": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": ["low", "high", "auto", "original"] + } + }, + "typed_choice_values": { + "type": "boolean" + }, + "partial_refusals": { + "type": "boolean" + }, + "maximum_questions": { + "type": "integer", + "minimum": 1 + }, + "maximum_images": { + "type": "integer", + "minimum": 1 + }, + "minimum_choices": { + "type": "integer", + "minimum": 1 + }, + "maximum_choices": { + "type": "integer", + "minimum": 1 + }, + "minimum_score_levels": { + "type": "integer", + "minimum": 1 + }, + "maximum_score_levels": { + "type": "integer", + "minimum": 1 + } + } + }, + "decision_card": { + "allOf": [ + { + "$ref": "#/$defs/card" + } + ], + "required": ["vendor", "underlying_model", "providers"], + "unevaluatedProperties": false, + "properties": { + "underlying_model": { + "type": "string", + "minLength": 1, + "description": "Canonical catalogue identity of the underlying model; this offering has its own interface and rates." + }, + "decision_contract": { + "$ref": "#/$defs/decision_contract" } } } diff --git a/scripts/build.py b/scripts/build.py index 2957775..8d762c1 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -36,7 +36,16 @@ def sections() -> list[str]: - """The models.json sections, in the order the schema requires them.""" + """Declared family sections, with optional families after existing required ones.""" + with MODELS_SCHEMA.open(encoding="utf-8") as f: + schema = json.load(f) + return schema["required"] + [ + name for name in schema["properties"] + if name != "$schema" and name not in schema["required"] + ] + + +def required_sections() -> list[str]: with MODELS_SCHEMA.open(encoding="utf-8") as f: return json.load(f)["required"] @@ -53,7 +62,7 @@ def build() -> tuple[dict[Path, bytes], list[str]]: labs = load_labs(MODELS) names = sections() outputs = { - MODELS_JSON: dumps(assemble(cards, names, schema_id())).encode(), + MODELS_JSON: dumps(assemble(cards, names, schema_id(), required_sections())).encode(), LABS: dumps(assemble_labs(labs)).encode(), } for lab in labs: diff --git a/scripts/cards.py b/scripts/cards.py index 5a10464..68dd9d0 100644 --- a/scripts/cards.py +++ b/scripts/cards.py @@ -213,10 +213,15 @@ def project(card: Card) -> dict[str, Any]: return data -def assemble(cards: list[Card], sections: list[str], schema_id: str | None = None) -> dict[str, Any]: +def assemble( + cards: list[Card], sections: list[str], schema_id: str | None = None, + required_sections: list[str] | None = None, +) -> dict[str, Any]: """Build models.json: its `$schema` first, then sections in schema order, cards sorted by id.""" facts: dict[str, Any] = {"$schema": schema_id} if schema_id else {} - facts.update({section: {} for section in sections}) + required = sections if required_sections is None else required_sections + used = {card.data.get("section") for card in cards} + facts.update({section: {} for section in sections if section in required or section in used}) for card in sorted(cards, key=lambda c: str(c.data.get("id", ""))): section = card.data.get("section") if section not in facts: diff --git a/scripts/check.py b/scripts/check.py index 11f5057..7abed67 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -51,7 +51,7 @@ # Families whose cards hold intrinsic facts plus per-provider bindings. # A primary-provider choice on such a card is product policy. -MULTI_PROVIDER_SECTIONS = ("image.models", "video.models") +MULTI_PROVIDER_SECTIONS = ("image.models", "video.models", "decision.models") # Known violations carried over unchanged from the source data. Each entry is # (section, card id, key). An exception that no longer matches the data is @@ -94,6 +94,7 @@ def is_calendar_date(value: str) -> bool: SOURCED_OBJECTS = ( "distribution", + "decision_contract", "output_token_calculator", "vision_tokenization", "capabilities", @@ -160,6 +161,8 @@ def numeric_range_errors(value: dict[str, Any], where: str) -> Iterator[str]: ("minimum", "maximum"), ("min_items", "max_items"), ("min_length", "max_length"), ("min_edge", "max_edge"), ("min_pixels", "max_pixels"), ("min", "max"), + ("minimum_choices", "maximum_choices"), + ("minimum_score_levels", "maximum_score_levels"), ]: lo, hi = value.get(lower), value.get(upper) if isinstance(lo, (int, float)) and isinstance(hi, (int, float)) and lo > hi: diff --git a/tests/test_check.py b/tests/test_check.py index d684ae8..3d0c55c 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -134,7 +134,7 @@ def test_hand_edited_labs_are_reported_out_of_date(): def test_cards_round_trip_every_null(): - assembled = cards.assemble(CARDS, SECTIONS, check.build.schema_id()) + assembled = cards.assemble(CARDS, SECTIONS, check.build.schema_id(), check.build.required_sections()) assert assembled == MODELS_JSON @@ -544,3 +544,84 @@ def test_singleton_operation_can_record_sourced_endpoint_parameters(): facts = copy.deepcopy(MODELS_JSON) facts["three_d.rig_check.operation"]["endpoints"] = [copy.deepcopy(ENDPOINT)] assert check.check(facts, SCHEMA) == [] + + +# Optional decision offerings use their own interface and do not inherit chat facts. + +def decision_fixture(): + return { + "id": "openai/decision-example", "label": "Decision example", "vendor": "openai", + "underlying_model": "openai/gpt-6-luna", + "release": {"date": None, "basis": "provider_endpoint", "source_url": "https://example.com/decisions"}, + "reference_provider": "openai", + "providers": { + "openai": { + "id": "native-model", "provider": "openai", "source_url": "https://example.com/decisions", + "decision_contract": { + "source_url": "https://example.com/decisions", "question_types": ["predicate", "choice", "score"], + "minimum_choices": 2, "maximum_choices": 255, "typed_choice_values": True, + }, + }, + }, + } + + +def decision_facts(): + facts = copy.deepcopy(MODELS_JSON) + facts["decision.models"] = {"openai/decision-example": decision_fixture()} + return facts + + +def test_optional_decision_family_preserves_prior_snapshots(): + facts = copy.deepcopy(MODELS_JSON) + facts.pop("decision.models", None) + assert check.check(facts, SCHEMA) == [] + assert "decision.models" not in check.build.required_sections() + + +def test_decision_card_has_independent_identity_and_provider_contract(): + assert check.check(decision_facts(), SCHEMA) == [] + + +@pytest.mark.parametrize("field", ["context_window", "output_limit", "cost", "tool_call", "multimodal"]) +def test_decision_card_rejects_inherited_chat_fields(field): + facts = decision_facts() + facts["decision.models"]["openai/decision-example"][field] = 1 + assert any(field in e for e in check.check(facts, SCHEMA)) + + +def test_decision_card_requires_a_vendor_underlying_model_and_binding(): + for field in ("vendor", "underlying_model", "providers"): + facts = decision_facts() + facts["decision.models"]["openai/decision-example"].pop(field) + assert check.check(facts, SCHEMA) + facts = decision_facts() + facts["decision.models"]["openai/decision-example"]["providers"] = {} + assert any("no provider bindings" in e for e in check.check(facts, SCHEMA)) + + +def test_decision_contract_bounds_and_source_are_checked(): + facts = decision_facts() + contract = facts["decision.models"]["openai/decision-example"]["providers"]["openai"]["decision_contract"] + contract["minimum_choices"] = 256 + assert any("minimum_choices exceeds maximum_choices" in e for e in check.check(facts, SCHEMA)) + contract["minimum_choices"] = 2 + contract["source_url"] = "https://user:password@example.com/decisions" + assert any("without credentials" in e for e in check.check(facts, SCHEMA)) + + +def test_decision_contract_rejects_unknown_vocabulary(): + facts = decision_facts() + contract = facts["decision.models"]["openai/decision-example"]["providers"]["openai"]["decision_contract"] + contract["question_types"] = ["chat"] + assert any("question_types" in e for e in check.check(facts, SCHEMA)) + + +def test_optional_family_is_generated_only_when_authored(): + required = check.build.required_sections() + old = [c for c in CARDS if c.data.get("section") != "decision.models"] + assert "decision.models" not in cards.assemble(old, SECTIONS, check.build.schema_id(), required) + example = cards.Card("openai/decision-example", "card.toml", {"section": "decision.models", **decision_fixture()}) + assert list(cards.layout_errors(old + [example], SECTIONS)) == [] + assembled = cards.assemble(old + [example], SECTIONS, check.build.schema_id(), required) + assert assembled["decision.models"]["openai/decision-example"] == decision_fixture()