From ca2fc50338ea986a63c31eff97fa3c340a62c1fc Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 09:12:11 +0000 Subject: [PATCH 01/48] feat(system): expose raw detected capability for model meta resolution Model meta gallery entries express hardware fallback through candidate ordering rather than a capability map, so they need the undecorated detected capability string without Capability's default/cpu fallback chain. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- pkg/system/capabilities.go | 11 +++++++++++ pkg/system/capabilities_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/pkg/system/capabilities.go b/pkg/system/capabilities.go index 034f1d0111ea..9e74a2e8b2fc 100644 --- a/pkg/system/capabilities.go +++ b/pkg/system/capabilities.go @@ -63,6 +63,17 @@ func (s *SystemState) CapabilityFilterDisabled() bool { return s.getSystemCapabilities() == disableCapability } +// ReportedCapability returns the raw detected capability string (e.g. "metal", +// "nvidia-cuda-12", "default") with no map-membership fallback applied. +// +// Why this exists alongside Capability: Capability resolves against a caller +// supplied map and falls back to "default" then "cpu" when the detected value +// is absent. Model meta entries express fallback through candidate ordering +// instead, so they need the undecorated value. +func (s *SystemState) ReportedCapability() string { + return s.getSystemCapabilities() +} + func (s *SystemState) Capability(capMap map[string]string) string { reportedCapability := s.getSystemCapabilities() diff --git a/pkg/system/capabilities_test.go b/pkg/system/capabilities_test.go index 1bb13b6478de..117e3ffdb9a1 100644 --- a/pkg/system/capabilities_test.go +++ b/pkg/system/capabilities_test.go @@ -164,3 +164,28 @@ var _ = Describe("CapabilityFilterDisabled", func() { Expect(s.IsBackendCompatible("cpu-whisperx", "quay.io/cpu")).To(BeTrue()) }) }) + +var _ = Describe("ReportedCapability", func() { + var previous string + + BeforeEach(func() { + previous = os.Getenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") + }) + + AfterEach(func() { + Expect(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", previous)).To(Succeed()) + }) + + It("returns the forced capability verbatim without map fallback", func() { + Expect(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "nvidia-cuda-12")).To(Succeed()) + state := &SystemState{} + Expect(state.ReportedCapability()).To(Equal("nvidia-cuda-12")) + }) + + It("does not fall back to default when the capability is unusual", func() { + Expect(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "metal")).To(Succeed()) + state := &SystemState{} + Expect(state.ReportedCapability()).To(Equal("metal")) + Expect(state.ReportedCapability()).NotTo(Equal("default")) + }) +}) From ad79ec28fb77adac149e9fbe0dd7ea48d8b9af32 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 09:52:19 +0000 Subject: [PATCH 02/48] refactor(system): drop duplicate capability accessor, cover DetectedCapability ReportedCapability was added with a body identical to the existing DetectedCapability. Keep one accessor and move the specs onto it, since DetectedCapability had no direct coverage of its no-fallback behavior. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- pkg/system/capabilities.go | 19 +++++++------------ pkg/system/capabilities_test.go | 8 ++++---- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/pkg/system/capabilities.go b/pkg/system/capabilities.go index 9e74a2e8b2fc..9d0d3c3c452a 100644 --- a/pkg/system/capabilities.go +++ b/pkg/system/capabilities.go @@ -63,17 +63,6 @@ func (s *SystemState) CapabilityFilterDisabled() bool { return s.getSystemCapabilities() == disableCapability } -// ReportedCapability returns the raw detected capability string (e.g. "metal", -// "nvidia-cuda-12", "default") with no map-membership fallback applied. -// -// Why this exists alongside Capability: Capability resolves against a caller -// supplied map and falls back to "default" then "cpu" when the detected value -// is absent. Model meta entries express fallback through candidate ordering -// instead, so they need the undecorated value. -func (s *SystemState) ReportedCapability() string { - return s.getSystemCapabilities() -} - func (s *SystemState) Capability(capMap map[string]string) string { reportedCapability := s.getSystemCapabilities() @@ -217,8 +206,14 @@ func (s *SystemState) BackendPreferenceTokens() []string { } } -// DetectedCapability returns the detected system capability string. +// DetectedCapability returns the raw detected capability string (e.g. "metal", +// "nvidia-cuda-12", "default") with no map-membership fallback applied. // This can be used by the UI to display what capability was detected. +// +// Why this exists alongside Capability: Capability resolves against a caller +// supplied map and falls back to "default" then "cpu" when the detected value +// is absent from that map. Callers that express fallback themselves, such as +// model meta entries ordering their own candidates, need the undecorated value. func (s *SystemState) DetectedCapability() string { return s.getSystemCapabilities() } diff --git a/pkg/system/capabilities_test.go b/pkg/system/capabilities_test.go index 117e3ffdb9a1..4701bbca0c59 100644 --- a/pkg/system/capabilities_test.go +++ b/pkg/system/capabilities_test.go @@ -165,7 +165,7 @@ var _ = Describe("CapabilityFilterDisabled", func() { }) }) -var _ = Describe("ReportedCapability", func() { +var _ = Describe("DetectedCapability", func() { var previous string BeforeEach(func() { @@ -179,13 +179,13 @@ var _ = Describe("ReportedCapability", func() { It("returns the forced capability verbatim without map fallback", func() { Expect(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "nvidia-cuda-12")).To(Succeed()) state := &SystemState{} - Expect(state.ReportedCapability()).To(Equal("nvidia-cuda-12")) + Expect(state.DetectedCapability()).To(Equal("nvidia-cuda-12")) }) It("does not fall back to default when the capability is unusual", func() { Expect(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", "metal")).To(Succeed()) state := &SystemState{} - Expect(state.ReportedCapability()).To(Equal("metal")) - Expect(state.ReportedCapability()).NotTo(Equal("default")) + Expect(state.DetectedCapability()).To(Equal("metal")) + Expect(state.DetectedCapability()).NotTo(Equal("default")) }) }) From cd6f58461848e3da6f4d336048bb29b87d78c0b6 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 09:56:27 +0000 Subject: [PATCH 03/48] feat(vram): parse IEC binary size suffixes (KiB..PiB) ParseSizeString accepted only SI suffixes, so a "20GiB" floor was rejected outright. Model and VRAM sizes are conventionally quoted in IEC units, and silently reading GiB as GB would understate a floor by about 7%. Purely additive: these inputs previously returned an unknown-suffix error. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- pkg/vram/estimate.go | 17 +++++++++++++++-- pkg/vram/hf_estimate_test.go | 5 +++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/vram/estimate.go b/pkg/vram/estimate.go index c91004a4bcd2..faf3f11995d3 100644 --- a/pkg/vram/estimate.go +++ b/pkg/vram/estimate.go @@ -169,8 +169,9 @@ func EstimateMultiContext(ctx context.Context, files []FileInput, contextSizes [ } // ParseSizeString parses a human-readable size string (e.g. "500MB", "14.5 GB", "2tb") -// into bytes. Supports B, KB, MB, GB, TB, PB (case-insensitive, space optional). -// Uses SI units (1 KB = 1000 B). +// into bytes. Supports B, KB, MB, GB, TB, PB (case-insensitive, space optional) +// as SI units (1 KB = 1000 B), and the IEC suffixes KiB, MiB, GiB, TiB, PiB as +// binary units (1 KiB = 1024 B). func ParseSizeString(s string) (uint64, error) { s = strings.TrimSpace(s) if s == "" { @@ -212,6 +213,18 @@ func ParseSizeString(s string) (uint64, error) { multiplier = 1000 * 1000 * 1000 * 1000 case "P", "PB": multiplier = 1000 * 1000 * 1000 * 1000 * 1000 + // IEC binary suffixes. Model and VRAM sizes are conventionally quoted in + // these, so treating "GiB" as "GB" would understate a floor by 7%. + case "KIB": + multiplier = 1024 + case "MIB": + multiplier = 1024 * 1024 + case "GIB": + multiplier = 1024 * 1024 * 1024 + case "TIB": + multiplier = 1024 * 1024 * 1024 * 1024 + case "PIB": + multiplier = 1024 * 1024 * 1024 * 1024 * 1024 default: return 0, fmt.Errorf("unknown size suffix: %q", suffix) } diff --git a/pkg/vram/hf_estimate_test.go b/pkg/vram/hf_estimate_test.go index 345d9ec24707..d0b66e9e97b5 100644 --- a/pkg/vram/hf_estimate_test.go +++ b/pkg/vram/hf_estimate_test.go @@ -25,6 +25,11 @@ var _ = Describe("ParseSizeString", func() { Entry("short suffix 100M", "100M", uint64(100_000_000)), Entry("short suffix 2G", "2G", uint64(2_000_000_000)), Entry("short suffix 1K", "1K", uint64(1_000)), + Entry("binary 1KiB", "1KiB", uint64(1024)), + Entry("binary 512MiB", "512MiB", uint64(512*1024*1024)), + Entry("binary 6GiB", "6GiB", uint64(6*1024*1024*1024)), + Entry("binary 2TiB", "2TiB", uint64(2*1024*1024*1024*1024)), + Entry("binary 1.5 gib lowercase with space", "1.5 gib", uint64(1.5*1024*1024*1024)), ) DescribeTable("invalid sizes", From 6caf0b40f9f90952b7ef4a5481cc74a1dda864d2 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 09:56:34 +0000 Subject: [PATCH 04/48] feat(gallery): add Candidate type for meta model entries Candidate is one option in a meta entry's ordered variant list. It names a concrete gallery entry and declares when that entry suits the host. EffectiveMinVRAM resolves the VRAM floor, letting an authored min_vram win over a nightly-inferred one. An unparseable floor errors instead of being treated as absent: swallowing a typo would turn a constrained candidate into an unconstrained one and select a too-large variant rather than fail loudly. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/gallery/candidate.go | 47 ++++++++++++++++++++++++++ core/gallery/resolve_candidate_test.go | 40 ++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 core/gallery/candidate.go create mode 100644 core/gallery/resolve_candidate_test.go diff --git a/core/gallery/candidate.go b/core/gallery/candidate.go new file mode 100644 index 000000000000..cede3d934f17 --- /dev/null +++ b/core/gallery/candidate.go @@ -0,0 +1,47 @@ +package gallery + +import ( + "fmt" + + "github.com/mudler/LocalAI/pkg/vram" +) + +// Candidate is one option in a meta model entry's ordered candidate list. It +// references an existing concrete gallery entry by name and declares the +// conditions under which that entry is the right choice for the host. +type Candidate struct { + // Model is the name of a concrete (non-meta) gallery entry. + Model string `json:"model" yaml:"model"` + // Capability, when set, must equal the host's reported capability + // (e.g. "metal", "nvidia-cuda-12"). Empty matches any host. + Capability string `json:"capability,omitempty" yaml:"capability,omitempty"` + // MinVRAM is the authored VRAM floor (e.g. "20GiB"). Authoritative: + // inference never overwrites it. + MinVRAM string `json:"min_vram,omitempty" yaml:"min_vram,omitempty"` + + // The fields below are denormalized by the nightly job for display and + // lint. They are never authored by hand and never affect what gets + // installed, because installation reads the concrete entry live. + Backend string `json:"backend,omitempty" yaml:"backend,omitempty"` + Quantization string `json:"quantization,omitempty" yaml:"quantization,omitempty"` + InferredMinVRAM string `json:"inferred_min_vram,omitempty" yaml:"inferred_min_vram,omitempty"` +} + +// EffectiveMinVRAM returns the VRAM floor in bytes for this candidate and +// whether a floor is declared at all. An authored MinVRAM wins over an +// inferred one. A candidate with no floor declares no requirement and always +// passes the VRAM check. +func (c Candidate) EffectiveMinVRAM() (uint64, bool, error) { + raw := c.MinVRAM + if raw == "" { + raw = c.InferredMinVRAM + } + if raw == "" { + return 0, false, nil + } + bytes, err := vram.ParseSizeString(raw) + if err != nil { + return 0, false, fmt.Errorf("candidate %q has an unparseable min_vram %q: %w", c.Model, raw, err) + } + return bytes, true, nil +} diff --git a/core/gallery/resolve_candidate_test.go b/core/gallery/resolve_candidate_test.go new file mode 100644 index 000000000000..77a48f3b54e7 --- /dev/null +++ b/core/gallery/resolve_candidate_test.go @@ -0,0 +1,40 @@ +package gallery_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/gallery" +) + +var _ = Describe("Candidate.EffectiveMinVRAM", func() { + It("reports no floor when neither authored nor inferred", func() { + c := gallery.Candidate{Model: "x"} + bytes, declared, err := c.EffectiveMinVRAM() + Expect(err).ToNot(HaveOccurred()) + Expect(declared).To(BeFalse()) + Expect(bytes).To(Equal(uint64(0))) + }) + + It("uses the inferred value when nothing is authored", func() { + c := gallery.Candidate{Model: "x", InferredMinVRAM: "6GiB"} + bytes, declared, err := c.EffectiveMinVRAM() + Expect(err).ToNot(HaveOccurred()) + Expect(declared).To(BeTrue()) + Expect(bytes).To(Equal(uint64(6 * 1024 * 1024 * 1024))) + }) + + It("lets an authored value win over an inferred one", func() { + c := gallery.Candidate{Model: "x", MinVRAM: "20GiB", InferredMinVRAM: "6GiB"} + bytes, declared, err := c.EffectiveMinVRAM() + Expect(err).ToNot(HaveOccurred()) + Expect(declared).To(BeTrue()) + Expect(bytes).To(Equal(uint64(20 * 1024 * 1024 * 1024))) + }) + + It("errors on an unparseable floor rather than silently treating it as absent", func() { + c := gallery.Candidate{Model: "x", MinVRAM: "twenty gigs"} + _, _, err := c.EffectiveMinVRAM() + Expect(err).To(HaveOccurred()) + }) +}) From 48821a00c773c606e59c5dac000bee85c6acec55 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 10:04:24 +0000 Subject: [PATCH 05/48] feat(gallery): add hardware-aware model variant resolver Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/gallery/resolve_candidate.go | 84 ++++++++++++++++++++++++++ core/gallery/resolve_candidate_test.go | 68 +++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 core/gallery/resolve_candidate.go diff --git a/core/gallery/resolve_candidate.go b/core/gallery/resolve_candidate.go new file mode 100644 index 000000000000..c8574590ac3a --- /dev/null +++ b/core/gallery/resolve_candidate.go @@ -0,0 +1,84 @@ +package gallery + +import ( + "errors" + "fmt" + "strings" +) + +var ( + // ErrNoCandidateMatch is returned when no candidate satisfies the host. + ErrNoCandidateMatch = errors.New("no model variant matches this system") + // ErrPinNotFound is returned when a pinned variant is absent from the list. + ErrPinNotFound = errors.New("pinned model variant not found") +) + +// ResolveEnv describes the host properties a candidate is matched against. +// +// It is a plain struct rather than a *SystemState so the resolver stays pure +// and testable across hardware shapes without touching the machine. +type ResolveEnv struct { + // Capability is the raw value from SystemState.DetectedCapability(). + Capability string + // VRAM is total available VRAM in bytes, already capped by the operator + // VRAM budget because the cap is applied inside detection. + VRAM uint64 +} + +// ResolveCandidate returns the first candidate the host satisfies, or the +// pinned candidate when pin is non-empty. +// +// Why first-match rather than best-match: the list order is the policy. It is +// authored deliberately and lint-checked, which keeps selection explainable +// (you can read the list top to bottom) and keeps fallback expressible without +// a scoring function nobody can predict. +func ResolveCandidate(candidates []Candidate, env ResolveEnv, pin string) (Candidate, error) { + if pin != "" { + for _, c := range candidates { + if strings.EqualFold(c.Model, pin) { + return c, nil + } + } + return Candidate{}, fmt.Errorf("%w: %q is not among the %d variants of this model", ErrPinNotFound, pin, len(candidates)) + } + + reasons := make([]string, 0, len(candidates)) + for _, c := range candidates { + if c.Capability != "" && c.Capability != env.Capability { + reasons = append(reasons, fmt.Sprintf("%s requires capability %q", c.Model, c.Capability)) + continue + } + + floor, declared, err := c.EffectiveMinVRAM() + if err != nil { + return Candidate{}, err + } + // A candidate with no declared floor states no requirement and + // passes. Lint guarantees only the final last-resort candidate is + // in that position for the official gallery. + if declared && env.VRAM < floor { + reasons = append(reasons, fmt.Sprintf("%s needs %s VRAM", c.Model, humanBytes(floor))) + continue + } + + return c, nil + } + + return Candidate{}, fmt.Errorf( + "%w: capability %q with %s VRAM available; candidates: %s", + ErrNoCandidateMatch, env.Capability, humanBytes(env.VRAM), strings.Join(reasons, "; "), + ) +} + +func humanBytes(b uint64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%dB", b) + } + div, exp := uint64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f%ciB", float64(b)/float64(div), "KMGTPE"[exp]) +} diff --git a/core/gallery/resolve_candidate_test.go b/core/gallery/resolve_candidate_test.go index 77a48f3b54e7..4ecdc6238236 100644 --- a/core/gallery/resolve_candidate_test.go +++ b/core/gallery/resolve_candidate_test.go @@ -38,3 +38,71 @@ var _ = Describe("Candidate.EffectiveMinVRAM", func() { Expect(err).To(HaveOccurred()) }) }) + +var _ = Describe("ResolveCandidate", func() { + gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 } + + list := []gallery.Candidate{ + {Model: "m-mlx", Capability: "metal", MinVRAM: "16GiB"}, + {Model: "m-vllm", Capability: "nvidia", MinVRAM: "20GiB"}, + {Model: "m-q8", MinVRAM: "12GiB"}, + {Model: "m-q4", MinVRAM: "6GiB"}, + {Model: "m-q4-cpu"}, + } + + It("picks the capability-matched candidate when VRAM allows", func() { + got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(got.Model).To(Equal("m-vllm")) + }) + + It("skips a capability match whose VRAM floor does not fit", func() { + got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(8)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(got.Model).To(Equal("m-q4")) + }) + + It("ignores candidates whose capability does not match the host", func() { + got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "metal", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(got.Model).To(Equal("m-mlx")) + }) + + It("falls through to the unconstrained last resort on a tiny host", func() { + got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "default", VRAM: gib(2)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(got.Model).To(Equal("m-q4-cpu")) + }) + + It("honors a pin even when the host would have chosen otherwise", func() { + got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "m-q8") + Expect(err).ToNot(HaveOccurred()) + Expect(got.Model).To(Equal("m-q8")) + }) + + It("fails loudly when the pin names an absent candidate", func() { + _, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "m-gone") + Expect(err).To(MatchError(gallery.ErrPinNotFound)) + Expect(err.Error()).To(ContainSubstring("m-gone")) + }) + + It("reports the host state when nothing matches", func() { + constrained := []gallery.Candidate{{Model: "big", MinVRAM: "80GiB"}} + _, err := gallery.ResolveCandidate(constrained, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).To(MatchError(gallery.ErrNoCandidateMatch)) + Expect(err.Error()).To(ContainSubstring("default")) + Expect(err.Error()).To(ContainSubstring("big")) + }) + + It("propagates an unparseable floor instead of treating it as unconstrained", func() { + bad := []gallery.Candidate{{Model: "bad", MinVRAM: "lots"}} + _, err := gallery.ResolveCandidate(bad, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("bad")) + }) + + It("rejects an empty candidate list", func() { + _, err := gallery.ResolveCandidate(nil, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).To(MatchError(gallery.ErrNoCandidateMatch)) + }) +}) From ba33c21567d27388086c795bb23c0dbb20502a39 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 10:10:55 +0000 Subject: [PATCH 06/48] feat(gallery): allow gallery model entries to declare variant candidates A gallery entry with a non-empty candidates list is a meta entry: it names an ordered list of concrete entries and resolves to the first one the host can satisfy, instead of describing model files directly. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/gallery/meta_install_test.go | 43 +++++++++++++++++++++++++++++++ core/gallery/models_types.go | 10 +++++++ 2 files changed, 53 insertions(+) create mode 100644 core/gallery/meta_install_test.go diff --git a/core/gallery/meta_install_test.go b/core/gallery/meta_install_test.go new file mode 100644 index 000000000000..158f2aaf56a7 --- /dev/null +++ b/core/gallery/meta_install_test.go @@ -0,0 +1,43 @@ +package gallery_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/mudler/LocalAI/core/gallery" +) + +var _ = Describe("GalleryModel meta entries", func() { + It("is not meta when it has no candidates", func() { + m := gallery.GalleryModel{} + m.Name = "plain" + Expect(m.IsMeta()).To(BeFalse()) + }) + + It("is meta when it has candidates", func() { + m := gallery.GalleryModel{Candidates: []gallery.Candidate{{Model: "x"}}} + Expect(m.IsMeta()).To(BeTrue()) + }) + + It("parses a candidate list from gallery YAML in order", func() { + var m gallery.GalleryModel + err := yaml.Unmarshal([]byte(` +name: qwen3-8b +url: "github:example/repo/qwen3-8b-gguf-q4.yaml@master" +candidates: + - model: qwen3-8b-vllm-awq + capability: nvidia + min_vram: 20GiB + - model: qwen3-8b-gguf-q4 +`), &m) + Expect(err).ToNot(HaveOccurred()) + Expect(m.IsMeta()).To(BeTrue()) + Expect(m.Candidates).To(HaveLen(2)) + Expect(m.Candidates[0].Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(m.Candidates[0].Capability).To(Equal("nvidia")) + Expect(m.Candidates[0].MinVRAM).To(Equal("20GiB")) + Expect(m.Candidates[1].Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(m.Candidates[1].Capability).To(BeEmpty()) + }) +}) diff --git a/core/gallery/models_types.go b/core/gallery/models_types.go index 24e32e03e80b..b50dfbcc144a 100644 --- a/core/gallery/models_types.go +++ b/core/gallery/models_types.go @@ -15,6 +15,10 @@ type GalleryModel struct { ConfigFile map[string]any `json:"config_file,omitempty" yaml:"config_file,omitempty"` // Overrides are used to override the configuration of the model located at URL Overrides map[string]any `json:"overrides,omitempty" yaml:"overrides,omitempty"` + // Candidates, when non-empty, makes this a meta entry: an ordered list of + // concrete gallery entries, the first of which the host satisfies is what + // gets installed. Mirrors GalleryBackend's capabilities map one level up. + Candidates []Candidate `json:"candidates,omitempty" yaml:"candidates,omitempty"` } func (m *GalleryModel) GetInstalled() bool { @@ -45,6 +49,12 @@ func (m GalleryModel) ID() string { return fmt.Sprintf("%s@%s", m.Gallery.Name, m.Name) } +// IsMeta reports whether this entry resolves to one of several concrete +// entries based on host hardware, rather than describing files directly. +func (m GalleryModel) IsMeta() bool { + return len(m.Candidates) > 0 +} + func (m *GalleryModel) GetTags() []string { return m.Tags } From 49fa504160415c64c409c5f65597ac829d3ab432 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 10:19:47 +0000 Subject: [PATCH 07/48] feat(gallery): resolve meta model entries to hardware-appropriate variants at install Meta gallery entries carry an ordered candidate list; at install time the first candidate the host satisfies is resolved and its payload installed under the meta's name, so the model keeps a stable name regardless of which variant backs it. The resolution is recorded in the installed gallery config so a reinstall honors a prior pin and operators can see the backing variant. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/gallery/meta_install_test.go | 74 +++++++++++++++++++++++ core/gallery/model_artifacts.go | 9 +++ core/gallery/models.go | 98 ++++++++++++++++++++++++++++++- 3 files changed, 179 insertions(+), 2 deletions(-) diff --git a/core/gallery/meta_install_test.go b/core/gallery/meta_install_test.go index 158f2aaf56a7..cdada35c001b 100644 --- a/core/gallery/meta_install_test.go +++ b/core/gallery/meta_install_test.go @@ -41,3 +41,77 @@ candidates: Expect(m.Candidates[1].Capability).To(BeEmpty()) }) }) + +var _ = Describe("ResolveMetaModel", func() { + gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 } + + newModel := func(name, url, description, icon string) *gallery.GalleryModel { + m := &gallery.GalleryModel{} + m.Name = name + m.URL = url + m.Description = description + m.Icon = icon + return m + } + + var models []*gallery.GalleryModel + var meta *gallery.GalleryModel + + BeforeEach(func() { + concreteVLLM := newModel("qwen3-8b-vllm-awq", "file://vllm.yaml", "AWQ variant", "vllm.png") + concreteGGUF := newModel("qwen3-8b-gguf-q4", "file://gguf.yaml", "GGUF variant", "gguf.png") + meta = newModel("qwen3-8b", "file://gguf.yaml", "Qwen3 8B", "qwen.png") + meta.Tags = []string{"llm"} + meta.Candidates = []gallery.Candidate{ + {Model: "qwen3-8b-vllm-awq", Capability: "nvidia", MinVRAM: "20GiB"}, + {Model: "qwen3-8b-gguf-q4"}, + } + models = []*gallery.GalleryModel{concreteVLLM, concreteGGUF, meta} + }) + + It("installs the concrete payload under the meta's name", func() { + resolved, candidate, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(candidate.Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(resolved.Name).To(Equal("qwen3-8b")) + Expect(resolved.URL).To(Equal("file://vllm.yaml")) + }) + + It("presents the meta's metadata, not the variant's", func() { + resolved, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(resolved.Description).To(Equal("Qwen3 8B")) + Expect(resolved.Icon).To(Equal("qwen.png")) + Expect(resolved.Tags).To(ConsistOf("llm")) + }) + + It("keeps the name stable when a variant is pinned", func() { + resolved, candidate, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "qwen3-8b-gguf-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.Name).To(Equal("qwen3-8b")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) + }) + + It("errors when a candidate references a missing entry", func() { + meta.Candidates = []gallery.Candidate{{Model: "does-not-exist"}} + _, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does-not-exist")) + }) + + It("refuses a candidate that is itself a meta entry", func() { + nested := newModel("nested", "", "", "") + nested.Candidates = []gallery.Candidate{{Model: "qwen3-8b-gguf-q4"}} + models = append(models, nested) + meta.Candidates = []gallery.Candidate{{Model: "nested"}} + _, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("nested")) + }) + + It("surfaces a bad pin", func() { + _, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "nope") + Expect(err).To(MatchError(gallery.ErrPinNotFound)) + }) +}) diff --git a/core/gallery/model_artifacts.go b/core/gallery/model_artifacts.go index 07924d9dfc16..01de02d032ba 100644 --- a/core/gallery/model_artifacts.go +++ b/core/gallery/model_artifacts.go @@ -19,6 +19,7 @@ type ArtifactMaterializer interface { type installOptions struct { materializer ArtifactMaterializer + variant string } type InstallOption func(*installOptions) @@ -31,6 +32,14 @@ func WithArtifactMaterializer(materializer ArtifactMaterializer) InstallOption { } } +// WithVariant pins a meta model entry to a specific candidate by name, +// bypassing hardware-based resolution. Ignored for non-meta entries. +func WithVariant(variant string) InstallOption { + return func(options *installOptions) { + options.variant = variant + } +} + func applyInstallOptions(options ...InstallOption) installOptions { result := installOptions{materializer: modelartifacts.NewDefaultManager()} for _, option := range options { diff --git a/core/gallery/models.go b/core/gallery/models.go index 4d39cc0cc7a4..88c81e48acd0 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -60,6 +60,13 @@ type ModelConfig struct { ConfigFile string `yaml:"config_file"` Files []File `yaml:"files"` PromptTemplates []PromptTemplate `yaml:"prompt_templates"` + + // The fields below record how a meta entry was resolved, so a reinstall + // or upgrade can honor the same pin and so operators can see which + // variant a stable model name is actually backed by. + MetaName string `yaml:"meta_name,omitempty"` + ResolvedVariant string `yaml:"resolved_variant,omitempty"` + PinnedVariant string `yaml:"pinned_variant,omitempty"` } type File struct { @@ -73,6 +80,50 @@ type PromptTemplate struct { Content string `yaml:"content"` } +// ResolveMetaModel turns a meta gallery entry into the concrete entry that +// should be installed on this host, returning it renamed to the meta's name +// and carrying the meta's presentation metadata. +// +// Why the metadata split: the payload (url, config_file, files, overrides) +// must come from the variant because that is what actually gets downloaded, +// while the presentation (name, description, icon, tags) must come from the +// meta so the installed model presents as the model rather than the variant. +func ResolveMetaModel(models []*GalleryModel, meta *GalleryModel, env ResolveEnv, pin string) (*GalleryModel, Candidate, error) { + candidate, err := ResolveCandidate(meta.Candidates, env, pin) + if err != nil { + return nil, Candidate{}, fmt.Errorf("resolving variant for model %q: %w", meta.Name, err) + } + + // A pin is an operator override and deliberately bypasses the hardware + // checks, but a silent bypass makes a later out-of-memory failure + // impossible to trace back to the pin, so it is recorded loudly here. + if pin != "" { + if floor, declared, verr := candidate.EffectiveMinVRAM(); verr == nil && declared && env.VRAM < floor { + xlog.Warn("Pinned model variant declares more VRAM than this system reports; installing anyway because the pin overrides hardware resolution", + "model", meta.Name, "variant", candidate.Model, "required_vram", floor, "available_vram", env.VRAM) + } + } + + concrete := FindGalleryElement(models, candidate.Model) + if concrete == nil { + return nil, Candidate{}, fmt.Errorf("model %q references variant %q which does not exist in any configured gallery", meta.Name, candidate.Model) + } + if concrete.IsMeta() { + return nil, Candidate{}, fmt.Errorf("model %q references variant %q which is itself a meta entry; meta entries may not nest", meta.Name, candidate.Model) + } + + resolved := *concrete + resolved.Name = meta.Name + resolved.Description = meta.Description + resolved.Icon = meta.Icon + resolved.License = meta.License + resolved.URLs = meta.URLs + resolved.Tags = meta.Tags + resolved.Candidates = nil + + return &resolved, candidate, nil +} + // Installs a model from the gallery func InstallModelFromGallery( ctx context.Context, @@ -81,7 +132,9 @@ func InstallModelFromGallery( modelLoader *model.ModelLoader, name string, req GalleryModel, downloadStatus func(string, string, string, float64), enforceScan, automaticallyInstallBackend, requireBackendIntegrity bool, options ...InstallOption) error { - applyModel := func(model *GalleryModel) error { + installOpts := applyInstallOptions(options...) + + applyModel := func(model *GalleryModel, record *ModelConfig) error { name = strings.ReplaceAll(name, string(os.PathSeparator), "__") var config ModelConfig @@ -113,6 +166,12 @@ func InstallModelFromGallery( return fmt.Errorf("invalid gallery model %+v", model) } + if record != nil { + config.MetaName = record.MetaName + config.ResolvedVariant = record.ResolvedVariant + config.PinnedVariant = record.PinnedVariant + } + installName := model.Name if req.Name != "" { installName = req.Name @@ -157,7 +216,42 @@ func InstallModelFromGallery( return fmt.Errorf("no model found with name %q", name) } - return applyModel(model) + // Meta-ness is checked before anything looks at the URL: a meta entry also + // carries a url as a fallback for older LocalAI releases that do not + // understand candidates, so carrying both is normal and meta wins here. + if !model.IsMeta() { + return applyModel(model, nil) + } + + pin := installOpts.variant + // A previously recorded pin survives reinstalls and upgrades, so a user + // who deliberately chose a variant is not silently re-resolved onto a + // different one by a hardware or gallery change. + if pin == "" { + if previous, err := GetLocalModelConfiguration(systemState.Model.ModelsPath, model.Name); err == nil && previous != nil { + pin = previous.PinnedVariant + } + } + + env := ResolveEnv{ + Capability: systemState.DetectedCapability(), + VRAM: systemState.VRAM, + } + + resolved, candidate, err := ResolveMetaModel(models, model, env, pin) + if err != nil { + return err + } + + xlog.Info("Resolved meta model to variant", + "model", model.Name, "variant", candidate.Model, + "capability", env.Capability, "vram", env.VRAM, "pinned", pin != "") + + return applyModel(resolved, &ModelConfig{ + MetaName: model.Name, + ResolvedVariant: candidate.Model, + PinnedVariant: pin, + }) } func InstallModel(ctx context.Context, systemState *system.SystemState, nameOverride string, galleryConfig *ModelConfig, configOverrides map[string]any, downloadStatus func(string, string, string, float64), enforceScan bool, options ...InstallOption) (*lconfig.ModelConfig, error) { From 47f37fa80d55d90a539b6ca34e46160c459c1720 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 10:36:04 +0000 Subject: [PATCH 08/48] fix(gallery): key meta pin recall on the installed name and detach resolved entries Six review findings on the meta-entry install path. Pin recall was keyed on the gallery entry name while applyModel writes the record under the install name (req.Name when supplied), so a meta installed under a custom name with a pin lost that pin on reinstall and was silently re-resolved onto a different variant, possibly swapping its backend. Compute the install name with applyModel's own precedence before the recall. ResolveMetaModel returned a shallow struct copy, so the resolved entry's Overrides aliased the gallery entry's map and the install path's in-place mergo merge wrote the caller's request into the shared catalog. Detach Overrides, ConfigFile, AdditionalFiles, URLs and Tags. Not exploitable today only because this path re-unmarshals the gallery per call, which is a property nobody should have to rely on. Also: overlay the meta's name onto the persisted config for meta installs so the gallery file no longer records the variant's name; move the pinned-VRAM warning below the variant validation so a pin naming a nonexistent entry does not warn about VRAM before failing for an unrelated reason; and stop seeding config.URLs in the config_file branch, which duplicated every declared URL. Add seven network-free specs driving InstallModelFromGallery with a meta entry: variant payload wins over the meta's legacy url fallback, the resolution record round-trips to disk, a pin is recorded and honored on reinstall including under a custom install name, and the resolved entry does not alias the gallery's maps. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/gallery/meta_install_test.go | 191 ++++++++++++++++++++++++++++++ core/gallery/models.go | 55 +++++++-- 2 files changed, 234 insertions(+), 12 deletions(-) diff --git a/core/gallery/meta_install_test.go b/core/gallery/meta_install_test.go index cdada35c001b..5d441bb5a3ce 100644 --- a/core/gallery/meta_install_test.go +++ b/core/gallery/meta_install_test.go @@ -1,11 +1,17 @@ package gallery_test import ( + "context" + "os" + "path/filepath" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "gopkg.in/yaml.v3" + "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/system" ) var _ = Describe("GalleryModel meta entries", func() { @@ -93,6 +99,28 @@ var _ = Describe("ResolveMetaModel", func() { Expect(resolved.URL).To(Equal("file://gguf.yaml")) }) + It("detaches the resolved entry from the gallery's own maps and slices", func() { + models[0].Overrides = map[string]any{"f16": true} + models[0].AdditionalFiles = []gallery.File{{Filename: "a.bin"}} + + resolved, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + + // The install path merges the caller's request overrides into this map in + // place, so aliasing the gallery's map would write one caller's request + // into the shared catalog and leak it into every later install. + resolved.Overrides["f16"] = false + resolved.Overrides["threads"] = 4 + Expect(models[0].Overrides).To(HaveKeyWithValue("f16", true)) + Expect(models[0].Overrides).ToNot(HaveKey("threads")) + + resolved.AdditionalFiles[0].Filename = "mutated.bin" + Expect(models[0].AdditionalFiles[0].Filename).To(Equal("a.bin")) + + resolved.Tags = append(resolved.Tags, "extra") + Expect(meta.Tags).To(ConsistOf("llm")) + }) + It("errors when a candidate references a missing entry", func() { meta.Candidates = []gallery.Candidate{{Model: "does-not-exist"}} _, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") @@ -115,3 +143,166 @@ var _ = Describe("ResolveMetaModel", func() { Expect(err).To(MatchError(gallery.ErrPinNotFound)) }) }) + +var _ = Describe("InstallModelFromGallery with meta entries", func() { + var tempdir string + var galleries []config.Gallery + var systemState *system.SystemState + + // The variants are described with an inline config_file rather than a URL so + // the whole install runs off the local filesystem with no network access. + // The meta entry keeps a url as well, because a real meta entry carries one + // as a fallback for older LocalAI releases that do not understand candidates, + // and installing that fallback instead of a variant is exactly the regression + // these specs guard against. + newGallery := func(meta gallery.GalleryModel, variants ...gallery.GalleryModel) { + fallback := gallery.ModelConfig{ + Name: "legacy-fallback", + Description: "legacy fallback payload", + ConfigFile: "backend: fallback-backend\n", + } + fallbackYAML, err := yaml.Marshal(fallback) + Expect(err).ToNot(HaveOccurred()) + fallbackPath := filepath.Join(tempdir, "fallback.yaml") + Expect(os.WriteFile(fallbackPath, fallbackYAML, 0600)).To(Succeed()) + + meta.URL = "file://" + fallbackPath + entries := append([]gallery.GalleryModel{meta}, variants...) + + out, err := yaml.Marshal(entries) + Expect(err).ToNot(HaveOccurred()) + galleryPath := filepath.Join(tempdir, "gallery.yaml") + Expect(os.WriteFile(galleryPath, out, 0600)).To(Succeed()) + + galleries = []config.Gallery{{Name: "test", URL: "file://" + galleryPath}} + } + + variant := func(name, backend string) gallery.GalleryModel { + m := gallery.GalleryModel{ConfigFile: map[string]any{"backend": backend}} + m.Name = name + m.Description = "variant " + name + return m + } + + metaEntry := func(name string, candidates ...string) gallery.GalleryModel { + m := gallery.GalleryModel{} + m.Name = name + m.Description = "the meta entry" + m.Icon = "meta.png" + for _, c := range candidates { + m.Candidates = append(m.Candidates, gallery.Candidate{Model: c}) + } + return m + } + + install := func(name string, req gallery.GalleryModel, options ...gallery.InstallOption) error { + return gallery.InstallModelFromGallery( + context.TODO(), galleries, []config.Gallery{}, systemState, nil, + name, req, func(string, string, string, float64) {}, false, false, false, options...) + } + + installedBackend := func(name string) string { + dat, err := os.ReadFile(filepath.Join(tempdir, name+".yaml")) + Expect(err).ToNot(HaveOccurred()) + content := map[string]any{} + Expect(yaml.Unmarshal(dat, &content)).To(Succeed()) + return content["backend"].(string) + } + + BeforeEach(func() { + var err error + tempdir, err = os.MkdirTemp("", "meta-install") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { Expect(os.RemoveAll(tempdir)).To(Succeed()) }) + + systemState, err = system.GetSystemState(system.WithModelPath(tempdir)) + Expect(err).ToNot(HaveOccurred()) + + newGallery( + metaEntry("qwen3-8b", "qwen3-8b-variant-a", "qwen3-8b-variant-b"), + variant("qwen3-8b-variant-a", "variant-a-backend"), + variant("qwen3-8b-variant-b", "variant-b-backend"), + ) + }) + + It("installs the resolved variant's payload, not the meta's url fallback", func() { + Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) + + // If meta-ness stopped winning over the url, this would be + // "fallback-backend" and every meta entry would silently install the + // legacy payload instead of a hardware-appropriate variant. + Expect(installedBackend("qwen3-8b")).To(Equal("variant-a-backend")) + }) + + It("round-trips the resolution record to disk under the meta's name", func() { + Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) + + record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") + Expect(err).ToNot(HaveOccurred()) + Expect(record.MetaName).To(Equal("qwen3-8b")) + Expect(record.ResolvedVariant).To(Equal("qwen3-8b-variant-a")) + Expect(record.PinnedVariant).To(BeEmpty()) + Expect(record.Name).To(Equal("qwen3-8b")) + Expect(record.Description).To(Equal("the meta entry")) + }) + + It("records a pin and honors it on a plain reinstall", func() { + Expect(install("qwen3-8b", gallery.GalleryModel{}, gallery.WithVariant("qwen3-8b-variant-b"))).To(Succeed()) + + record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") + Expect(err).ToNot(HaveOccurred()) + Expect(record.PinnedVariant).To(Equal("qwen3-8b-variant-b")) + Expect(record.ResolvedVariant).To(Equal("qwen3-8b-variant-b")) + Expect(installedBackend("qwen3-8b")).To(Equal("variant-b-backend")) + + // No WithVariant this time: hardware resolution would pick variant-a, so + // only the recalled pin can keep this on variant-b. + Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) + Expect(installedBackend("qwen3-8b")).To(Equal("variant-b-backend")) + + record, err = gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") + Expect(err).ToNot(HaveOccurred()) + Expect(record.PinnedVariant).To(Equal("qwen3-8b-variant-b")) + }) + + It("honors a pin recorded under a custom install name", func() { + req := gallery.GalleryModel{} + req.Name = "prod-llm" + + Expect(install("qwen3-8b", req, gallery.WithVariant("qwen3-8b-variant-b"))).To(Succeed()) + + // The pin is written under the installed name, so the recall must read it + // back under that name too and not under the gallery entry's name. + record, err := gallery.GetLocalModelConfiguration(tempdir, "prod-llm") + Expect(err).ToNot(HaveOccurred()) + Expect(record.PinnedVariant).To(Equal("qwen3-8b-variant-b")) + Expect(record.MetaName).To(Equal("qwen3-8b")) + Expect(installedBackend("prod-llm")).To(Equal("variant-b-backend")) + + Expect(install("qwen3-8b", req)).To(Succeed()) + Expect(installedBackend("prod-llm")).To(Equal("variant-b-backend")) + }) + + It("does not write the caller's overrides back into the gallery entry", func() { + req := gallery.GalleryModel{Overrides: map[string]any{"f16": true}} + Expect(install("qwen3-8b", req)).To(Succeed()) + + models, err := gallery.AvailableGalleryModels(galleries, systemState) + Expect(err).ToNot(HaveOccurred()) + entry := gallery.FindGalleryElement(models, "qwen3-8b-variant-a") + Expect(entry).ToNot(BeNil()) + Expect(entry.Overrides).ToNot(HaveKey("f16")) + }) + + It("writes each declared url only once into the persisted gallery file", func() { + meta := metaEntry("qwen3-8b", "qwen3-8b-variant-a") + meta.URLs = []string{"https://example.invalid/qwen3"} + newGallery(meta, variant("qwen3-8b-variant-a", "variant-a-backend")) + + Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) + + record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") + Expect(err).ToNot(HaveOccurred()) + Expect(record.URLs).To(ConsistOf("https://example.invalid/qwen3")) + }) +}) diff --git a/core/gallery/models.go b/core/gallery/models.go index 88c81e48acd0..0b1e65028ca0 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "maps" "net/url" "os" "path/filepath" @@ -94,9 +95,20 @@ func ResolveMetaModel(models []*GalleryModel, meta *GalleryModel, env ResolveEnv return nil, Candidate{}, fmt.Errorf("resolving variant for model %q: %w", meta.Name, err) } + concrete := FindGalleryElement(models, candidate.Model) + if concrete == nil { + return nil, Candidate{}, fmt.Errorf("model %q references variant %q which does not exist in any configured gallery", meta.Name, candidate.Model) + } + if concrete.IsMeta() { + return nil, Candidate{}, fmt.Errorf("model %q references variant %q which is itself a meta entry; meta entries may not nest", meta.Name, candidate.Model) + } + // A pin is an operator override and deliberately bypasses the hardware // checks, but a silent bypass makes a later out-of-memory failure // impossible to trace back to the pin, so it is recorded loudly here. + // It is warned about only once the pin is known to name a real, installable + // entry, otherwise a pin naming a listed-but-nonexistent variant would warn + // about VRAM and then fail for an entirely unrelated reason. if pin != "" { if floor, declared, verr := candidate.EffectiveMinVRAM(); verr == nil && declared && env.VRAM < floor { xlog.Warn("Pinned model variant declares more VRAM than this system reports; installing anyway because the pin overrides hardware resolution", @@ -104,23 +116,25 @@ func ResolveMetaModel(models []*GalleryModel, meta *GalleryModel, env ResolveEnv } } - concrete := FindGalleryElement(models, candidate.Model) - if concrete == nil { - return nil, Candidate{}, fmt.Errorf("model %q references variant %q which does not exist in any configured gallery", meta.Name, candidate.Model) - } - if concrete.IsMeta() { - return nil, Candidate{}, fmt.Errorf("model %q references variant %q which is itself a meta entry; meta entries may not nest", meta.Name, candidate.Model) - } - resolved := *concrete resolved.Name = meta.Name resolved.Description = meta.Description resolved.Icon = meta.Icon resolved.License = meta.License - resolved.URLs = meta.URLs - resolved.Tags = meta.Tags resolved.Candidates = nil + // The struct copy above is shallow, so every reference-typed field still + // aliases the gallery's own entries. The install path mutates Overrides in + // place (mergo merges the caller's request overrides into it) and appends to + // the URL and tag slices, which would write the caller's request into the + // gallery catalog itself and leak between installs the moment this path + // reads from a cached, long-lived gallery listing. Detach them here. + resolved.Overrides = maps.Clone(concrete.Overrides) + resolved.ConfigFile = maps.Clone(concrete.ConfigFile) + resolved.AdditionalFiles = slices.Clone(concrete.AdditionalFiles) + resolved.URLs = slices.Clone(meta.URLs) + resolved.Tags = slices.Clone(meta.Tags) + return &resolved, candidate, nil } @@ -157,9 +171,11 @@ func InstallModelFromGallery( ConfigFile: string(reYamlConfig), Description: model.Description, License: model.License, - URLs: model.URLs, Name: model.Name, Files: make([]File, 0), // Real values get added below, must be blank + // URLs are deliberately not seeded here: they are appended once + // below for both branches, and seeding them too would write every + // URL twice into the persisted gallery file. // Prompt Template Skipped for now - I expect in this mode that they will be delivered as files. } } else { @@ -170,6 +186,10 @@ func InstallModelFromGallery( config.MetaName = record.MetaName config.ResolvedVariant = record.ResolvedVariant config.PinnedVariant = record.PinnedVariant + // The variant's own name would otherwise be persisted here, which + // contradicts the whole point of a meta entry: the model is known by + // the meta's stable name regardless of which variant backs it. + config.Name = record.MetaName } installName := model.Name @@ -227,8 +247,19 @@ func InstallModelFromGallery( // A previously recorded pin survives reinstalls and upgrades, so a user // who deliberately chose a variant is not silently re-resolved onto a // different one by a hardware or gallery change. + // + // The record is keyed by the name the model was installed under, not by the + // gallery entry name: applyModel writes it to ._gallery_.yaml, + // where installName is req.Name whenever the caller supplied one. Reading it + // back under the meta's own name would miss the record for every custom-named + // install and silently re-resolve a deliberately pinned model onto a + // different variant, possibly swapping its backend. if pin == "" { - if previous, err := GetLocalModelConfiguration(systemState.Model.ModelsPath, model.Name); err == nil && previous != nil { + installName := model.Name + if req.Name != "" { + installName = req.Name + } + if previous, err := GetLocalModelConfiguration(systemState.Model.ModelsPath, installName); err == nil && previous != nil { pin = previous.PinnedVariant } } From 4835c4373f06284e6d6d4b8db1c801594b120cd2 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 10:53:27 +0000 Subject: [PATCH 09/48] fix(gallery): deep-copy meta overrides and make two specs functional ResolveMetaModel detached the resolved entry's Overrides and ConfigFile with maps.Clone, which only copies the top level. Gallery overrides are nested in practice (parameters.model is near-universal) and the install path merges the caller's request with mergo.WithOverride, which recurses into nested maps and overwrites them in place, so the gallery entry's own inner maps were still reachable and still got rewritten by the last caller to install. Copy both maps all the way down instead, recursing through the container shapes a YAML decoder produces. ConfigFile is not mutated on the install path today, but it carries the same kind of nested payload and leaving it shallowly cloned would invite the bug back. Also fix two specs that passed whether or not their target fix was present: - "does not write the caller's overrides back into the gallery entry" re-read the catalog from disk, which re-unmarshals fresh structs and so cannot observe in-memory aliasing. It now asserts against the in-memory gallery entry and drives the real mergo merge. - "round-trips the resolution record to disk under the meta's name" asserted a name that is already correct in the config_file branch. It now drives the url branch via a file:// fixture, where the meta-name overlay actually applies. Both were verified red by reverting their fix. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/gallery/meta_install_test.go | 84 +++++++++++++++++++++++++++---- core/gallery/models.go | 51 +++++++++++++++++-- 2 files changed, 121 insertions(+), 14 deletions(-) diff --git a/core/gallery/meta_install_test.go b/core/gallery/meta_install_test.go index 5d441bb5a3ce..df730b3cb21c 100644 --- a/core/gallery/meta_install_test.go +++ b/core/gallery/meta_install_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" + "dario.cat/mergo" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "gopkg.in/yaml.v3" @@ -121,6 +122,43 @@ var _ = Describe("ResolveMetaModel", func() { Expect(meta.Tags).To(ConsistOf("llm")) }) + It("detaches nested override maps from the gallery's own entry", func() { + models[0].Overrides = map[string]any{ + "parameters": map[string]any{"model": "real-variant.gguf"}, + "stopwords": []any{""}, + } + + resolved, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + + // Cloning only the top level would leave this inner map shared with the + // gallery entry, so writing through the resolved copy would rewrite the + // catalog's own payload. + resolved.Overrides["parameters"].(map[string]any)["model"] = "callers-choice.gguf" + resolved.Overrides["stopwords"].([]any)[0] = "<|im_end|>" + + Expect(models[0].Overrides["parameters"]).To(HaveKeyWithValue("model", "real-variant.gguf")) + Expect(models[0].Overrides["stopwords"]).To(Equal([]any{""})) + }) + + It("does not write the caller's overrides back into the gallery entry", func() { + models[0].Overrides = map[string]any{"parameters": map[string]any{"model": "real-variant.gguf"}} + + resolved, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + + // This is exactly what the install path does with the caller's request + // overrides, and mergo recurses into nested maps and overwrites them in + // place. Asserting against the in-memory catalog is the only way to + // observe the leak: re-reading the gallery from disk re-unmarshals fresh + // maps and would pass whether or not the resolved entry was detached. + requestOverrides := map[string]any{"parameters": map[string]any{"model": "callers-choice.gguf"}} + Expect(mergo.Merge(&resolved.Overrides, requestOverrides, mergo.WithOverride)).To(Succeed()) + + Expect(resolved.Overrides["parameters"]).To(HaveKeyWithValue("model", "callers-choice.gguf")) + Expect(models[0].Overrides["parameters"]).To(HaveKeyWithValue("model", "real-variant.gguf")) + }) + It("errors when a candidate references a missing entry", func() { meta.Candidates = []gallery.Candidate{{Model: "does-not-exist"}} _, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") @@ -184,6 +222,30 @@ var _ = Describe("InstallModelFromGallery with meta entries", func() { return m } + // urlVariant describes a variant through a url rather than an inline + // config_file. The distinction matters for the recorded name: the url branch + // reads a name out of the fetched config, and that name is the variant's own, + // so only the meta-name overlay can keep the record under the meta's name. + // The inline config_file branch seeds the name from the already-renamed + // resolved entry and so cannot observe the overlay at all. + urlVariant := func(name, backend string) gallery.GalleryModel { + payload := gallery.ModelConfig{ + Name: name, + Description: "variant " + name, + ConfigFile: "backend: " + backend + "\n", + } + out, err := yaml.Marshal(payload) + Expect(err).ToNot(HaveOccurred()) + payloadPath := filepath.Join(tempdir, "payload-"+name+".yaml") + Expect(os.WriteFile(payloadPath, out, 0600)).To(Succeed()) + + m := gallery.GalleryModel{} + m.Name = name + m.Description = "variant " + name + m.URL = "file://" + payloadPath + return m + } + metaEntry := func(name string, candidates ...string) gallery.GalleryModel { m := gallery.GalleryModel{} m.Name = name @@ -235,6 +297,16 @@ var _ = Describe("InstallModelFromGallery with meta entries", func() { }) It("round-trips the resolution record to disk under the meta's name", func() { + // The variant is described by url so its payload carries its own name. + // Without the meta-name overlay the record persists as + // "qwen3-8b-variant-a", and the stable name a meta entry exists to + // provide is lost the moment anything reads the record back. + newGallery( + metaEntry("qwen3-8b", "qwen3-8b-variant-a", "qwen3-8b-variant-b"), + urlVariant("qwen3-8b-variant-a", "variant-a-backend"), + urlVariant("qwen3-8b-variant-b", "variant-b-backend"), + ) + Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") @@ -244,6 +316,7 @@ var _ = Describe("InstallModelFromGallery with meta entries", func() { Expect(record.PinnedVariant).To(BeEmpty()) Expect(record.Name).To(Equal("qwen3-8b")) Expect(record.Description).To(Equal("the meta entry")) + Expect(installedBackend("qwen3-8b")).To(Equal("variant-a-backend")) }) It("records a pin and honors it on a plain reinstall", func() { @@ -283,17 +356,6 @@ var _ = Describe("InstallModelFromGallery with meta entries", func() { Expect(installedBackend("prod-llm")).To(Equal("variant-b-backend")) }) - It("does not write the caller's overrides back into the gallery entry", func() { - req := gallery.GalleryModel{Overrides: map[string]any{"f16": true}} - Expect(install("qwen3-8b", req)).To(Succeed()) - - models, err := gallery.AvailableGalleryModels(galleries, systemState) - Expect(err).ToNot(HaveOccurred()) - entry := gallery.FindGalleryElement(models, "qwen3-8b-variant-a") - Expect(entry).ToNot(BeNil()) - Expect(entry.Overrides).ToNot(HaveKey("f16")) - }) - It("writes each declared url only once into the persisted gallery file", func() { meta := metaEntry("qwen3-8b", "qwen3-8b-variant-a") meta.URLs = []string{"https://example.invalid/qwen3"} diff --git a/core/gallery/models.go b/core/gallery/models.go index 0b1e65028ca0..252dc539c806 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "maps" "net/url" "os" "path/filepath" @@ -129,8 +128,15 @@ func ResolveMetaModel(models []*GalleryModel, meta *GalleryModel, env ResolveEnv // the URL and tag slices, which would write the caller's request into the // gallery catalog itself and leak between installs the moment this path // reads from a cached, long-lived gallery listing. Detach them here. - resolved.Overrides = maps.Clone(concrete.Overrides) - resolved.ConfigFile = maps.Clone(concrete.ConfigFile) + // + // Overrides and ConfigFile are copied all the way down rather than cloned at + // the top level only: gallery overrides are nested in practice (a + // parameters.model map is near-universal), and mergo recurses into nested + // maps and overwrites them in place, so a top-level clone would still hand + // the caller the gallery's own inner maps. The slices below hold value types, + // so cloning them once fully detaches them. + resolved.Overrides = deepCopyStringMap(concrete.Overrides) + resolved.ConfigFile = deepCopyStringMap(concrete.ConfigFile) resolved.AdditionalFiles = slices.Clone(concrete.AdditionalFiles) resolved.URLs = slices.Clone(meta.URLs) resolved.Tags = slices.Clone(meta.Tags) @@ -138,6 +144,45 @@ func ResolveMetaModel(models []*GalleryModel, meta *GalleryModel, env ResolveEnv return &resolved, candidate, nil } +// deepCopyStringMap copies a decoded YAML map so no part of the result, at any +// depth, is reachable from the original. +func deepCopyStringMap(m map[string]any) map[string]any { + if m == nil { + return nil + } + out := make(map[string]any, len(m)) + for k, v := range m { + out[k] = deepCopyYAMLValue(v) + } + return out +} + +// deepCopyYAMLValue recurses through the only container shapes a YAML decoder +// produces. Scalars are returned as-is because they cannot be mutated through +// the copy. map[any]any is handled as well because gopkg.in/yaml.v2 decodes +// non-string keys into it, and gallery documents are not guaranteed to have +// passed through the v3 decoder. +func deepCopyYAMLValue(v any) any { + switch t := v.(type) { + case map[string]any: + return deepCopyStringMap(t) + case map[any]any: + out := make(map[any]any, len(t)) + for k, val := range t { + out[k] = deepCopyYAMLValue(val) + } + return out + case []any: + out := make([]any, len(t)) + for i, val := range t { + out[i] = deepCopyYAMLValue(val) + } + return out + default: + return v + } +} + // Installs a model from the gallery func InstallModelFromGallery( ctx context.Context, From 8ed8cf60e9fadce904ffdb80ba6698a460223ddd Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 10:58:34 +0000 Subject: [PATCH 10/48] test(gallery): lint meta model entry invariants in index.yaml Adds Ginkgo specs that parse the shipped gallery/index.yaml and enforce the invariants that keep meta entries safe: a legacy url fallback equal to the final candidate's url, references only to existing non-meta entries, a min_vram floor on every candidate but the last-resort one, a capability drawn only from the vocabulary the system can report, and descending VRAM floors within a capability group. The capability check is the only compensating control for a typo there. Candidate matching is a case-sensitive exact comparison against SystemState.DetectedCapability(), so an unknown value never matches and falls through silently instead of erroring. The vocabulary therefore mirrors the raw return set of getSystemCapabilities(), which notably excludes "cpu": that is a fallback key inside Capability(capMap) on the meta backend path, never a reported capability. A CPU-only host reports "default". These pass vacuously until the pilot meta entry lands; the guard is intentionally in place before the thing it guards. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/gallery/meta_lint_test.go | 130 +++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 core/gallery/meta_lint_test.go diff --git a/core/gallery/meta_lint_test.go b/core/gallery/meta_lint_test.go new file mode 100644 index 000000000000..38cf10f86388 --- /dev/null +++ b/core/gallery/meta_lint_test.go @@ -0,0 +1,130 @@ +package gallery_test + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/mudler/LocalAI/core/gallery" +) + +// knownCapabilities mirrors the values SystemState.DetectedCapability() can +// actually report, which is what the candidate resolver compares against with +// a case-sensitive exact match. A capability outside this set can never match, +// so a typo would silently make a candidate unreachable on every host. +// +// Note "cpu" is deliberately absent: it exists only as a fallback key inside +// SystemState.Capability(capMap) for meta backends, and is never a value +// getSystemCapabilities() returns. A CPU-only host reports "default". +var knownCapabilities = map[string]bool{ + "default": true, "metal": true, "darwin-x86": true, + "nvidia": true, "nvidia-cuda-12": true, "nvidia-cuda-13": true, + "nvidia-l4t": true, "nvidia-l4t-cuda-12": true, "nvidia-l4t-cuda-13": true, + "intel": true, "amd": true, "vulkan": true, +} + +var _ = Describe("gallery/index.yaml meta entry invariants", func() { + var entries []gallery.GalleryModel + var byName map[string]gallery.GalleryModel + + BeforeEach(func() { + data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) + Expect(err).ToNot(HaveOccurred()) + Expect(yaml.Unmarshal(data, &entries)).To(Succeed()) + + byName = map[string]gallery.GalleryModel{} + for _, e := range entries { + byName[e.Name] = e + } + }) + + It("gives every meta entry a legacy url and no inline payload", func() { + for _, e := range entries { + if !e.IsMeta() { + continue + } + // Released LocalAI versions ignore the candidates key. Without a + // url they would list the entry and install an empty model. + Expect(e.URL).ToNot(BeEmpty(), "meta entry %q needs a url fallback for older clients", e.Name) + Expect(e.ConfigFile).To(BeEmpty(), "meta entry %q must not carry an inline config_file", e.Name) + Expect(e.AdditionalFiles).To(BeEmpty(), "meta entry %q must not carry files", e.Name) + + last := e.Candidates[len(e.Candidates)-1] + fallback, ok := byName[last.Model] + Expect(ok).To(BeTrue(), "meta entry %q final candidate %q not found", e.Name, last.Model) + Expect(e.URL).To(Equal(fallback.URL), + "meta entry %q url must equal its final candidate %q url so old and new clients agree", e.Name, last.Model) + } + }) + + It("references only existing, non-meta entries", func() { + for _, e := range entries { + if !e.IsMeta() { + continue + } + for _, c := range e.Candidates { + target, ok := byName[c.Model] + Expect(ok).To(BeTrue(), "meta entry %q references unknown model %q", e.Name, c.Model) + Expect(target.IsMeta()).To(BeFalse(), "meta entry %q references meta entry %q; nesting is not allowed", e.Name, c.Model) + } + } + }) + + It("constrains every candidate except an unconstrained last resort", func() { + for _, e := range entries { + if !e.IsMeta() { + continue + } + for i, c := range e.Candidates { + _, declared, err := c.EffectiveMinVRAM() + Expect(err).ToNot(HaveOccurred(), "meta entry %q candidate %q has a bad min_vram", e.Name, c.Model) + + if i == len(e.Candidates)-1 { + Expect(declared).To(BeFalse(), "meta entry %q final candidate %q must be an unconstrained last resort", e.Name, c.Model) + Expect(c.Capability).To(BeEmpty(), "meta entry %q final candidate %q must not require a capability", e.Name, c.Model) + continue + } + Expect(declared).To(BeTrue(), "meta entry %q candidate %q needs a min_vram; the nightly job should have inferred one", e.Name, c.Model) + } + } + }) + + It("uses only capabilities the system can report", func() { + for _, e := range entries { + if !e.IsMeta() { + continue + } + for _, c := range e.Candidates { + if c.Capability == "" { + continue + } + Expect(knownCapabilities).To(HaveKey(c.Capability), + "meta entry %q candidate %q uses unknown capability %q", e.Name, c.Model, c.Capability) + } + } + }) + + It("orders candidates by descending VRAM within a capability group", func() { + for _, e := range entries { + if !e.IsMeta() { + continue + } + previous := map[string]uint64{} + for _, c := range e.Candidates { + floor, declared, err := c.EffectiveMinVRAM() + Expect(err).ToNot(HaveOccurred()) + if !declared { + continue + } + if prior, seen := previous[c.Capability]; seen { + Expect(floor).To(BeNumerically("<=", prior), + "meta entry %q candidate %q raises the VRAM floor after a lower one in the same capability group, so it can never be reached", e.Name, c.Model) + } + previous[c.Capability] = floor + } + } + }) +}) From 2794b31865dede4683bbb701654e775395b029e7 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 11:11:29 +0000 Subject: [PATCH 11/48] test(gallery): close coverage gaps in the meta entry lint The ordering invariant grouped candidates by capability and asserted floors descend within a group. A candidate with an EMPTY capability matches every host, so it does not belong in its own group: it dominates every later candidate whose floor is at or above its own, across capability groups. Track a running minimum floor over the unconditional candidates instead, which subsumes the old same-group check for the empty capability. Every spec skipped non-meta entries, so with zero meta entries in the index all five bodies were no-ops. Aligning GalleryModel.IsMeta() with GalleryBackend.IsMeta(), whose semantics are deliberately opposite, would have made all of them pass while checking nothing. Extract each invariant into a helper over a slice of entries returning the violations it finds, and cover those helpers with synthetic fixtures so the logic stays tested at zero meta entries. The index-driven specs are now a thin application of already proven logic. Also assert the index parses non-empty, report every violation in one run rather than aborting on the first, and parse the index once for the suite. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/gallery/meta_lint_test.go | 526 ++++++++++++++++++++++++++++----- 1 file changed, 453 insertions(+), 73 deletions(-) diff --git a/core/gallery/meta_lint_test.go b/core/gallery/meta_lint_test.go index 38cf10f86388..d964f14570ba 100644 --- a/core/gallery/meta_lint_test.go +++ b/core/gallery/meta_lint_test.go @@ -1,8 +1,11 @@ package gallery_test import ( + "fmt" "os" "path/filepath" + "strings" + "sync" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -26,105 +29,482 @@ var knownCapabilities = map[string]bool{ "intel": true, "amd": true, "vulkan": true, } -var _ = Describe("gallery/index.yaml meta entry invariants", func() { - var entries []gallery.GalleryModel - var byName map[string]gallery.GalleryModel +// metaViolation is one invariant breach found by a lint helper. Helpers return +// every breach they find instead of stopping at the first, so a single run +// names them all rather than forcing a fix-one-rerun-repeat cycle. +type metaViolation struct { + Entry string + Candidate string + Detail string +} - BeforeEach(func() { - data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) - Expect(err).ToNot(HaveOccurred()) - Expect(yaml.Unmarshal(data, &entries)).To(Succeed()) +func (v metaViolation) String() string { + if v.Candidate == "" { + return fmt.Sprintf("%s: %s", v.Entry, v.Detail) + } + return fmt.Sprintf("%s -> candidate %q: %s", v.Entry, v.Candidate, v.Detail) +} - byName = map[string]gallery.GalleryModel{} - for _, e := range entries { - byName[e.Name] = e +func formatViolations(violations []metaViolation) string { + lines := make([]string, 0, len(violations)) + for _, v := range violations { + lines = append(lines, " "+v.String()) + } + return "\n" + strings.Join(lines, "\n") +} + +func indexEntriesByName(entries []gallery.GalleryModel) map[string]gallery.GalleryModel { + byName := make(map[string]gallery.GalleryModel, len(entries)) + for _, e := range entries { + byName[e.Name] = e + } + return byName +} + +// checkMetaFallbackURL verifies that released LocalAI versions, which ignore +// the candidates key entirely, still install something sensible: the meta +// entry needs a url, and it must be the final candidate's url so old and new +// clients agree on the least demanding option. +func checkMetaFallbackURL(entries []gallery.GalleryModel) []metaViolation { + byName := indexEntriesByName(entries) + var violations []metaViolation + for _, e := range entries { + if !e.IsMeta() { + continue + } + if e.URL == "" { + violations = append(violations, metaViolation{Entry: e.Name, Detail: "needs a url fallback for older clients"}) + } + if len(e.ConfigFile) > 0 { + violations = append(violations, metaViolation{Entry: e.Name, Detail: "must not carry an inline config_file"}) + } + if len(e.AdditionalFiles) > 0 { + violations = append(violations, metaViolation{Entry: e.Name, Detail: "must not carry files"}) } - }) - It("gives every meta entry a legacy url and no inline payload", func() { - for _, e := range entries { - if !e.IsMeta() { - continue - } - // Released LocalAI versions ignore the candidates key. Without a - // url they would list the entry and install an empty model. - Expect(e.URL).ToNot(BeEmpty(), "meta entry %q needs a url fallback for older clients", e.Name) - Expect(e.ConfigFile).To(BeEmpty(), "meta entry %q must not carry an inline config_file", e.Name) - Expect(e.AdditionalFiles).To(BeEmpty(), "meta entry %q must not carry files", e.Name) - - last := e.Candidates[len(e.Candidates)-1] - fallback, ok := byName[last.Model] - Expect(ok).To(BeTrue(), "meta entry %q final candidate %q not found", e.Name, last.Model) - Expect(e.URL).To(Equal(fallback.URL), - "meta entry %q url must equal its final candidate %q url so old and new clients agree", e.Name, last.Model) + last := e.Candidates[len(e.Candidates)-1] + fallback, ok := byName[last.Model] + if !ok { + // checkMetaReferences owns reporting the dangling name; there is + // nothing to compare the url against here. + continue } - }) + if e.URL != fallback.URL { + violations = append(violations, metaViolation{ + Entry: e.Name, + Candidate: last.Model, + Detail: "meta url must equal its final candidate url so old and new clients agree", + }) + } + } + return violations +} - It("references only existing, non-meta entries", func() { - for _, e := range entries { - if !e.IsMeta() { +// checkMetaReferences verifies every candidate names an existing, concrete +// entry. Resolution is a single pass, so a nested meta reference would install +// an entry that carries no files. +func checkMetaReferences(entries []gallery.GalleryModel) []metaViolation { + byName := indexEntriesByName(entries) + var violations []metaViolation + for _, e := range entries { + if !e.IsMeta() { + continue + } + for _, c := range e.Candidates { + target, ok := byName[c.Model] + if !ok { + violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "references unknown model"}) continue } - for _, c := range e.Candidates { - target, ok := byName[c.Model] - Expect(ok).To(BeTrue(), "meta entry %q references unknown model %q", e.Name, c.Model) - Expect(target.IsMeta()).To(BeFalse(), "meta entry %q references meta entry %q; nesting is not allowed", e.Name, c.Model) + if target.IsMeta() { + violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "references a meta entry; nesting is not allowed"}) } } - }) + } + return violations +} - It("constrains every candidate except an unconstrained last resort", func() { - for _, e := range entries { - if !e.IsMeta() { +// checkMetaConstraints verifies that only the final candidate is an +// unconstrained last resort. An earlier candidate without a floor would +// capture every host and make everything after it dead. +func checkMetaConstraints(entries []gallery.GalleryModel) []metaViolation { + var violations []metaViolation + for _, e := range entries { + if !e.IsMeta() { + continue + } + for i, c := range e.Candidates { + _, declared, err := c.EffectiveMinVRAM() + if err != nil { + violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "has a bad min_vram: " + err.Error()}) continue } - for i, c := range e.Candidates { - _, declared, err := c.EffectiveMinVRAM() - Expect(err).ToNot(HaveOccurred(), "meta entry %q candidate %q has a bad min_vram", e.Name, c.Model) - - if i == len(e.Candidates)-1 { - Expect(declared).To(BeFalse(), "meta entry %q final candidate %q must be an unconstrained last resort", e.Name, c.Model) - Expect(c.Capability).To(BeEmpty(), "meta entry %q final candidate %q must not require a capability", e.Name, c.Model) - continue + + if i == len(e.Candidates)-1 { + if declared { + violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "final candidate must be an unconstrained last resort"}) + } + if c.Capability != "" { + violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "final candidate must not require a capability"}) } - Expect(declared).To(BeTrue(), "meta entry %q candidate %q needs a min_vram; the nightly job should have inferred one", e.Name, c.Model) + continue + } + if !declared { + violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "needs a min_vram; the nightly job should have inferred one"}) } } - }) + } + return violations +} - It("uses only capabilities the system can report", func() { - for _, e := range entries { - if !e.IsMeta() { +// checkMetaCapabilities verifies candidates only name capabilities the system +// can actually report, since the resolver compares them exactly. +func checkMetaCapabilities(entries []gallery.GalleryModel) []metaViolation { + var violations []metaViolation + for _, e := range entries { + if !e.IsMeta() { + continue + } + for _, c := range e.Candidates { + if c.Capability == "" { continue } - for _, c := range e.Candidates { - if c.Capability == "" { - continue - } - Expect(knownCapabilities).To(HaveKey(c.Capability), - "meta entry %q candidate %q uses unknown capability %q", e.Name, c.Model, c.Capability) + if !knownCapabilities[c.Capability] { + violations = append(violations, metaViolation{ + Entry: e.Name, + Candidate: c.Model, + Detail: fmt.Sprintf("uses unknown capability %q", c.Capability), + }) } } - }) + } + return violations +} + +// checkMetaOrdering verifies no candidate is shadowed by an earlier one. +// Selection is first-match over the authored order, so a candidate is dead +// whenever an earlier one matches every host this one would. +// +// Two shapes cause that. Within a single capability group the groups are +// mutually exclusive, so only a floor rising above an earlier floor is +// unreachable. A candidate with an EMPTY capability matches every host and so +// dominates ACROSS groups: every later candidate whose floor is at or above +// the running minimum unconditional floor is dead, whatever capability it asks +// for. Tracking that running minimum subsumes the same-group check for the +// empty capability. +func checkMetaOrdering(entries []gallery.GalleryModel) []metaViolation { + var violations []metaViolation + for _, e := range entries { + if !e.IsMeta() { + continue + } + previous := map[string]uint64{} + var unconditionalFloor uint64 + haveUnconditional := false - It("orders candidates by descending VRAM within a capability group", func() { - for _, e := range entries { - if !e.IsMeta() { + for _, c := range e.Candidates { + floor, declared, err := c.EffectiveMinVRAM() + if err != nil || !declared { + // Parse errors and absent floors belong to checkMetaConstraints. continue } - previous := map[string]uint64{} - for _, c := range e.Candidates { - floor, declared, err := c.EffectiveMinVRAM() - Expect(err).ToNot(HaveOccurred()) - if !declared { - continue - } - if prior, seen := previous[c.Capability]; seen { - Expect(floor).To(BeNumerically("<=", prior), - "meta entry %q candidate %q raises the VRAM floor after a lower one in the same capability group, so it can never be reached", e.Name, c.Model) + + switch { + case haveUnconditional && floor >= unconditionalFloor: + violations = append(violations, metaViolation{ + Entry: e.Name, + Candidate: c.Model, + Detail: fmt.Sprintf("is shadowed by an earlier candidate that matches any host at a %d byte floor, so it can never be reached", + unconditionalFloor), + }) + case c.Capability != "": + if prior, seen := previous[c.Capability]; seen && floor > prior { + violations = append(violations, metaViolation{ + Entry: e.Name, + Candidate: c.Model, + Detail: "raises the VRAM floor after a lower one in the same capability group, so it can never be reached", + }) } - previous[c.Capability] = floor } + + if c.Capability == "" && (!haveUnconditional || floor < unconditionalFloor) { + unconditionalFloor = floor + haveUnconditional = true + } + previous[c.Capability] = floor } + } + return violations +} + +// loadGalleryIndex parses gallery/index.yaml once for the whole suite. The +// index carries well over a thousand entries, so re-parsing it per spec is +// pure overhead. +var loadGalleryIndex = sync.OnceValues(func() ([]gallery.GalleryModel, error) { + data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) + if err != nil { + return nil, err + } + var entries []gallery.GalleryModel + if err := yaml.Unmarshal(data, &entries); err != nil { + return nil, err + } + return entries, nil +}) + +func concreteEntry(name, url string) gallery.GalleryModel { + e := gallery.GalleryModel{} + e.Name = name + e.URL = url + return e +} + +func metaEntry(name, url string, candidates ...gallery.Candidate) gallery.GalleryModel { + e := gallery.GalleryModel{Candidates: candidates} + e.Name = name + e.URL = url + return e +} + +// metaFixture builds a meta entry plus the concrete entries it references. +// Synthetic fixtures keep the invariant logic covered while gallery/index.yaml +// still holds zero meta entries; without them every index-driven spec below is +// a no-op that passes while checking nothing. +func metaFixture(meta gallery.GalleryModel, concrete ...gallery.GalleryModel) []gallery.GalleryModel { + return append([]gallery.GalleryModel{meta}, concrete...) +} + +var _ = Describe("meta entry lint helpers", func() { + // A model entry is meta solely by carrying candidates. GalleryBackend.IsMeta() + // has deliberately opposite semantics (it requires an EMPTY uri), so a + // well-meaning alignment of the two would make every helper skip every + // entry and pass silently. Assert the distinction directly. + It("treats an entry with candidates as meta even though it has a url", func() { + Expect(metaEntry("meta", "u://big", gallery.Candidate{Model: "big"}).IsMeta()).To(BeTrue()) + Expect(concreteEntry("big", "u://big").IsMeta()).To(BeFalse()) + }) + + It("passes every invariant on a valid meta entry", func() { + entries := metaFixture( + metaEntry("meta", "u://small", + gallery.Candidate{Model: "big", Capability: "nvidia", MinVRAM: "24GiB"}, + gallery.Candidate{Model: "mid", Capability: "nvidia", MinVRAM: "12GiB"}, + gallery.Candidate{Model: "metal-big", Capability: "metal", MinVRAM: "32GiB"}, + gallery.Candidate{Model: "small"}, + ), + concreteEntry("big", "u://big"), + concreteEntry("mid", "u://mid"), + concreteEntry("metal-big", "u://metal-big"), + concreteEntry("small", "u://small"), + ) + + Expect(checkMetaFallbackURL(entries)).To(BeEmpty()) + Expect(checkMetaReferences(entries)).To(BeEmpty()) + Expect(checkMetaConstraints(entries)).To(BeEmpty()) + Expect(checkMetaCapabilities(entries)).To(BeEmpty()) + Expect(checkMetaOrdering(entries)).To(BeEmpty()) + }) + + Describe("checkMetaOrdering", func() { + It("flags a candidate shadowed by an earlier unconditional candidate", func() { + // "a" matches any host at 8GiB, so the nvidia candidate at 20GiB is + // dead: every nvidia host clearing 20GiB already cleared 8GiB. + entries := metaFixture( + metaEntry("meta", "u://c", + gallery.Candidate{Model: "a", MinVRAM: "8GiB"}, + gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"}, + gallery.Candidate{Model: "c"}, + ), + concreteEntry("a", "u://a"), concreteEntry("b", "u://b"), concreteEntry("c", "u://c"), + ) + + violations := checkMetaOrdering(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("b")) + Expect(violations[0].Detail).To(ContainSubstring("shadowed by an earlier candidate that matches any host")) + }) + + It("flags a floor inversion inside one capability group", func() { + entries := metaFixture( + metaEntry("meta", "u://c", + gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"}, + gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"}, + gallery.Candidate{Model: "c"}, + ), + concreteEntry("a", "u://a"), concreteEntry("b", "u://b"), concreteEntry("c", "u://c"), + ) + + violations := checkMetaOrdering(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("b")) + Expect(violations[0].Detail).To(ContainSubstring("same capability group")) + }) + + It("keeps distinct capability groups independent", func() { + // A high metal floor after a low nvidia floor is fine: no host + // reports both capabilities. + entries := metaFixture( + metaEntry("meta", "u://c", + gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"}, + gallery.Candidate{Model: "b", Capability: "metal", MinVRAM: "32GiB"}, + gallery.Candidate{Model: "c"}, + ), + concreteEntry("a", "u://a"), concreteEntry("b", "u://b"), concreteEntry("c", "u://c"), + ) + + Expect(checkMetaOrdering(entries)).To(BeEmpty()) + }) + }) + + Describe("checkMetaConstraints", func() { + It("flags a non-final candidate with no floor", func() { + entries := metaFixture( + metaEntry("meta", "u://c", + gallery.Candidate{Model: "a", Capability: "nvidia"}, + gallery.Candidate{Model: "c"}, + ), + concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), + ) + + violations := checkMetaConstraints(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("a")) + Expect(violations[0].Detail).To(ContainSubstring("needs a min_vram")) + }) + + It("flags a final candidate that declares a floor", func() { + entries := metaFixture( + metaEntry("meta", "u://c", + gallery.Candidate{Model: "a", MinVRAM: "20GiB"}, + gallery.Candidate{Model: "c", MinVRAM: "8GiB"}, + ), + concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), + ) + + violations := checkMetaConstraints(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("c")) + Expect(violations[0].Detail).To(ContainSubstring("unconstrained last resort")) + }) + + It("flags a final candidate that requires a capability", func() { + entries := metaFixture( + metaEntry("meta", "u://c", + gallery.Candidate{Model: "a", MinVRAM: "20GiB"}, + gallery.Candidate{Model: "c", Capability: "nvidia"}, + ), + concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), + ) + + violations := checkMetaConstraints(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("c")) + Expect(violations[0].Detail).To(ContainSubstring("must not require a capability")) + }) + }) + + Describe("checkMetaReferences", func() { + It("flags a candidate naming an entry that does not exist", func() { + entries := metaFixture( + metaEntry("meta", "u://c", + gallery.Candidate{Model: "ghost", MinVRAM: "20GiB"}, + gallery.Candidate{Model: "c"}, + ), + concreteEntry("c", "u://c"), + ) + + violations := checkMetaReferences(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("ghost")) + Expect(violations[0].Detail).To(ContainSubstring("unknown model")) + }) + + It("flags a candidate naming another meta entry", func() { + entries := metaFixture( + metaEntry("meta", "u://c", + gallery.Candidate{Model: "nested", MinVRAM: "20GiB"}, + gallery.Candidate{Model: "c"}, + ), + metaEntry("nested", "u://c", gallery.Candidate{Model: "c"}), + concreteEntry("c", "u://c"), + ) + + violations := checkMetaReferences(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("nested")) + Expect(violations[0].Detail).To(ContainSubstring("nesting is not allowed")) + }) + }) + + Describe("checkMetaFallbackURL", func() { + It("flags a meta url that differs from its final candidate url", func() { + entries := metaFixture( + metaEntry("meta", "u://wrong", + gallery.Candidate{Model: "a", MinVRAM: "20GiB"}, + gallery.Candidate{Model: "c"}, + ), + concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), + ) + + violations := checkMetaFallbackURL(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("c")) + Expect(violations[0].Detail).To(ContainSubstring("must equal its final candidate url")) + }) + }) + + Describe("checkMetaCapabilities", func() { + It("flags a capability the system can never report", func() { + entries := metaFixture( + metaEntry("meta", "u://c", + gallery.Candidate{Model: "a", Capability: "cuda", MinVRAM: "20GiB"}, + gallery.Candidate{Model: "c"}, + ), + concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), + ) + + violations := checkMetaCapabilities(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("a")) + Expect(violations[0].Detail).To(ContainSubstring(`unknown capability "cuda"`)) + }) + }) +}) + +var _ = Describe("gallery/index.yaml meta entry invariants", Ordered, func() { + var entries []gallery.GalleryModel + + BeforeAll(func() { + var err error + entries, err = loadGalleryIndex() + Expect(err).ToNot(HaveOccurred()) + // A truncated or emptied index unmarshals cleanly and would make every + // spec below vacuously pass. + Expect(entries).ToNot(BeEmpty()) + }) + + It("gives every meta entry a legacy url and no inline payload", func() { + v := checkMetaFallbackURL(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("references only existing, non-meta entries", func() { + v := checkMetaReferences(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("constrains every candidate except an unconstrained last resort", func() { + v := checkMetaConstraints(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("uses only capabilities the system can report", func() { + v := checkMetaCapabilities(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("orders candidates so no candidate is shadowed by an earlier one", func() { + v := checkMetaOrdering(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) }) }) From 9ed4f6a0767ef6e0aa2a20a903a8a6807201e2f4 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 11:18:23 +0000 Subject: [PATCH 12/48] ci(gallery): add nightly denormalization of meta model candidates Fills the read-only backend, quantization and inferred_min_vram fields on meta gallery candidates and opens a PR, modeled on the existing checksum_checker job. Computing these needs network access, so it happens nightly rather than at install time. An authored min_vram is never modified: a human who measured a real load knows more than a pre-download estimate does. The index is rewritten via yaml.Node rather than a document round-trip. A full round-trip reflows all ~26k lines of gallery/index.yaml, which would bury the computed values and make the nightly PR unreviewable. The rewrite touches only the three derived keys, so authored styling survives and a run that computes nothing leaves the file untouched. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- .github/ci/gallery_denormalize.go | 258 +++++++++++++++++++++ .github/workflows/gallery-denormalize.yaml | 30 +++ 2 files changed, 288 insertions(+) create mode 100644 .github/ci/gallery_denormalize.go create mode 100644 .github/workflows/gallery-denormalize.yaml diff --git a/.github/ci/gallery_denormalize.go b/.github/ci/gallery_denormalize.go new file mode 100644 index 000000000000..12521a4c1629 --- /dev/null +++ b/.github/ci/gallery_denormalize.go @@ -0,0 +1,258 @@ +//go:build ignore + +// Command gallery_denormalize fills the read-only denormalized fields on meta +// model entries: backend, quantization, and inferred_min_vram. +// +// It never modifies an authored min_vram. Authored values are authoritative, +// because a human who measured a real load knows more than a pre-download +// estimate does. +package main + +import ( + "context" + "fmt" + "os" + "time" + + "gopkg.in/yaml.v3" + + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/vram" +) + +const estimateContext = 8192 + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "usage: gallery_denormalize ") + os.Exit(1) + } + path := os.Args[1] + + data, err := os.ReadFile(path) + if err != nil { + fmt.Fprintf(os.Stderr, "read %s: %v\n", path, err) + os.Exit(1) + } + + var entries []gallery.GalleryModel + if err := yaml.Unmarshal(data, &entries); err != nil { + fmt.Fprintf(os.Stderr, "parse %s: %v\n", path, err) + os.Exit(1) + } + + byName := map[string]gallery.GalleryModel{} + for _, e := range entries { + byName[e.Name] = e + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + failures := 0 + for i := range entries { + if !entries[i].IsMeta() { + continue + } + for j := range entries[i].Candidates { + c := &entries[i].Candidates[j] + + target, ok := byName[c.Model] + if !ok { + fmt.Fprintf(os.Stderr, "%s: candidate %q not found\n", entries[i].Name, c.Model) + failures++ + continue + } + + // The concrete entry's payload usually lives behind its url + // rather than inline, so fetch it. Metadata.Backend is populated + // at load time by the running server, not by parsing the index, + // so it is always empty here and cannot be copied. + resolved, err := gallery.GetGalleryConfigFromURLWithContext[gallery.ModelConfig](ctx, target.URL, "") + if err != nil { + fmt.Fprintf(os.Stderr, "%s: cannot fetch config for %q: %v\n", entries[i].Name, c.Model, err) + failures++ + continue + } + + c.Backend = backendOf(target, resolved) + c.Quantization = quantizationOf(target) + + // Authored values win, and the final unconstrained candidate is + // deliberately floorless, so neither gets an inferred value. + if c.MinVRAM != "" || j == len(entries[i].Candidates)-1 { + continue + } + + // Weight files can come from either side: a concrete index entry + // usually lists them itself (files:, i.e. AdditionalFiles) while + // the url it points at is only a base config, but some entries + // invert that. Install time downloads both, so estimate over both. + files := make([]vram.FileInput, 0, len(target.AdditionalFiles)+len(resolved.Files)) + for _, f := range target.AdditionalFiles { + files = append(files, vram.FileInput{URI: f.URI}) + } + for _, f := range resolved.Files { + files = append(files, vram.FileInput{URI: f.URI}) + } + + estimate, err := vram.EstimateModelMultiContext(ctx, vram.ModelEstimateInput{ + Files: files, + Size: target.Size, + }, []uint32{estimateContext}) + if err != nil || estimate.VRAMForContext(estimateContext) == 0 { + fmt.Fprintf(os.Stderr, + "%s: could not estimate min_vram for %q (%v); author one by hand\n", + entries[i].Name, c.Model, err) + failures++ + continue + } + c.InferredMinVRAM = fmt.Sprintf("%dMiB", estimate.VRAMForContext(estimateContext)/(1024*1024)) + } + } + + if err := writeCandidates(path, data, entries); err != nil { + fmt.Fprintf(os.Stderr, "write %s: %v\n", path, err) + os.Exit(1) + } + + if failures > 0 { + fmt.Fprintf(os.Stderr, "%d candidate(s) need a hand-authored min_vram\n", failures) + os.Exit(1) + } +} + +// writeCandidates writes back only the three derived keys on each candidate, +// leaving every other byte of the index alone. +// +// Marshalling the whole document back out would reflow all 1200+ entries +// (quoting, indentation, key order, line wrapping), burying the handful of +// real changes in reformatting noise and making the nightly PR unreviewable. +// Even re-encoding just the candidates subtree would restyle authored fields +// such as min_vram, so the rewrite reaches down to individual mapping keys. +// That also makes the job idempotent: a run that computes the same values +// leaves the file untouched and opens no PR. +func writeCandidates(path string, data []byte, entries []gallery.GalleryModel) error { + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil { + return fmt.Errorf("re-parse for node rewrite: %w", err) + } + if doc.Kind != yaml.DocumentNode || len(doc.Content) == 0 { + return fmt.Errorf("unexpected document shape") + } + root := doc.Content[0] + if root.Kind != yaml.SequenceNode { + return fmt.Errorf("index root is not a sequence") + } + if len(root.Content) != len(entries) { + return fmt.Errorf("entry count drift: %d nodes vs %d entries", len(root.Content), len(entries)) + } + + changed := false + for i, entryNode := range root.Content { + if !entries[i].IsMeta() || entryNode.Kind != yaml.MappingNode { + continue + } + + candidatesNode := mappingValue(entryNode, "candidates") + if candidatesNode == nil || candidatesNode.Kind != yaml.SequenceNode { + return fmt.Errorf("entry %q has candidates but no candidates sequence in the document", entries[i].Name) + } + if len(candidatesNode.Content) != len(entries[i].Candidates) { + return fmt.Errorf("entry %q candidate count drift: %d nodes vs %d parsed", + entries[i].Name, len(candidatesNode.Content), len(entries[i].Candidates)) + } + + for j, candidateNode := range candidatesNode.Content { + if candidateNode.Kind != yaml.MappingNode { + return fmt.Errorf("entry %q candidate %d is not a mapping", entries[i].Name, j) + } + c := entries[i].Candidates[j] + for _, kv := range []struct{ key, value string }{ + {"backend", c.Backend}, + {"quantization", c.Quantization}, + {"inferred_min_vram", c.InferredMinVRAM}, + } { + if setMappingValue(candidateNode, kv.key, kv.value) { + changed = true + } + } + } + } + + // Re-encoding an untouched document still normalizes indentation, so skip + // the write entirely when there was nothing to rewrite. + if !changed { + return nil + } + + out, err := yaml.Marshal(&doc) + if err != nil { + return fmt.Errorf("marshal: %w", err) + } + return os.WriteFile(path, out, 0644) +} + +// mappingValue returns the value node for key, or nil when absent. +func mappingValue(mapping *yaml.Node, key string) *yaml.Node { + for i := 0; i+1 < len(mapping.Content); i += 2 { + if mapping.Content[i].Value == key { + return mapping.Content[i+1] + } + } + return nil +} + +// setMappingValue sets key to value, appending the key when absent and dropping +// it when value is empty so a field that stops being derivable does not leave a +// stale number behind. It reports whether the document actually changed. +func setMappingValue(mapping *yaml.Node, key, value string) bool { + for i := 0; i+1 < len(mapping.Content); i += 2 { + if mapping.Content[i].Value != key { + continue + } + if value == "" { + mapping.Content = append(mapping.Content[:i], mapping.Content[i+2:]...) + return true + } + if mapping.Content[i+1].Value == value { + return false + } + mapping.Content[i+1] = scalarNode(value) + return true + } + if value == "" { + return false + } + mapping.Content = append(mapping.Content, scalarNode(key), scalarNode(value)) + return true +} + +func scalarNode(value string) *yaml.Node { + return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value} +} + +// backendOf mirrors the priority used by resolveBackend in +// core/gallery/backend_resolve.go: an entry's overrides win over whatever the +// referenced config file declares. +func backendOf(m gallery.GalleryModel, resolved gallery.ModelConfig) string { + if b, ok := m.Overrides["backend"].(string); ok && b != "" { + return b + } + var cfg struct { + Backend string `yaml:"backend"` + } + if err := yaml.Unmarshal([]byte(resolved.ConfigFile), &cfg); err == nil { + return cfg.Backend + } + return "" +} + +// quantizationOf reports the quantization declared by a concrete entry's +// overrides, empty when it declares none. +func quantizationOf(m gallery.GalleryModel) string { + if q, ok := m.Overrides["quantization"].(string); ok { + return q + } + return "" +} diff --git a/.github/workflows/gallery-denormalize.yaml b/.github/workflows/gallery-denormalize.yaml new file mode 100644 index 000000000000..14551d3b1044 --- /dev/null +++ b/.github/workflows/gallery-denormalize.yaml @@ -0,0 +1,30 @@ +name: Denormalize gallery meta entries +on: + schedule: + - cron: 0 22 * * * + workflow_dispatch: +jobs: + denormalize: + if: github.repository == 'mudler/LocalAI' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Denormalize meta candidates + run: go run ./.github/ci/gallery_denormalize.go gallery/index.yaml + - name: Create Pull Request + uses: peter-evans/create-pull-request@v8 + with: + token: ${{ secrets.UPDATE_BOT_TOKEN }} + push-to-fork: ci-forks/LocalAI + commit-message: ':arrow_up: Denormalize meta model candidates' + title: 'chore(model-gallery): :arrow_up: refresh meta candidate metadata' + branch: "update/gallery-denormalize" + body: | + Refreshes the denormalized `backend`, `quantization` and + `inferred_min_vram` fields on meta model entries. + + Authored `min_vram` values are never modified. + signoff: true From 98a0e3d7fa4061f1b06f20dde01bb9590dcab399 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 11:31:18 +0000 Subject: [PATCH 13/48] fix(ci): keep the gallery denormalize diff reviewable and self-healing The nightly denormalization job edits YAML nodes instead of round-tripping structs so its PR stays small enough for a human to review, but the write path undid that: yaml.Marshal re-encoded the node tree at yaml.v3's default 4-space indent and dropped the leading document marker, reflowing roughly 6000 lines around the handful of real changes. Encode through yaml.NewEncoder at the index's authored 2-space indent and restore the header. A write that changes three fields now changes three lines. Stale inferred_min_vram values were also never cleared. Both skip paths (an authored min_vram is present, or the candidate is the last resort) returned before touching the field, so a candidate that gained a floor or became the last resort after a reorder kept an inferred value that EffectiveMinVRAM reported as a real constraint, failing the meta lint with no way for the job to self-heal. Clear the field before both skips. The workflow discarded a whole night's work on any single failure: the program exits 1 when a candidate cannot be estimated, which aborted the job before the PR step, so one unreachable candidate blocked every other refresh indefinitely. Capture the status, open the PR with what was computed, mark the PR body as partial, and fail the run afterwards so the problem still surfaces. Also preserve the index's existing file mode instead of forcing 0644, and drop the redundant //go:build ignore tag, since Go already skips dot directories and the sibling modelslist.go carries no tag. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- .github/ci/gallery_denormalize.go | 66 +++++++++++++++++----- .github/workflows/gallery-denormalize.yaml | 28 ++++++++- 2 files changed, 79 insertions(+), 15 deletions(-) diff --git a/.github/ci/gallery_denormalize.go b/.github/ci/gallery_denormalize.go index 12521a4c1629..a650a64580e9 100644 --- a/.github/ci/gallery_denormalize.go +++ b/.github/ci/gallery_denormalize.go @@ -1,5 +1,3 @@ -//go:build ignore - // Command gallery_denormalize fills the read-only denormalized fields on meta // model entries: backend, quantization, and inferred_min_vram. // @@ -9,6 +7,7 @@ package main import ( + "bytes" "context" "fmt" "os" @@ -80,7 +79,12 @@ func main() { // Authored values win, and the final unconstrained candidate is // deliberately floorless, so neither gets an inferred value. + // Clearing first matters: a candidate that gained an authored + // min_vram, or that became the last resort after a reorder, would + // otherwise keep a stale inferred floor that makes + // EffectiveMinVRAM report a constraint the entry no longer has. if c.MinVRAM != "" || j == len(entries[i].Candidates)-1 { + c.InferredMinVRAM = "" continue } @@ -125,13 +129,14 @@ func main() { // writeCandidates writes back only the three derived keys on each candidate, // leaving every other byte of the index alone. // -// Marshalling the whole document back out would reflow all 1200+ entries -// (quoting, indentation, key order, line wrapping), burying the handful of -// real changes in reformatting noise and making the nightly PR unreviewable. -// Even re-encoding just the candidates subtree would restyle authored fields -// such as min_vram, so the rewrite reaches down to individual mapping keys. -// That also makes the job idempotent: a run that computes the same values -// leaves the file untouched and opens no PR. +// Round-tripping through []GalleryModel would reflow all 1200+ entries +// (quoting, key order, line wrapping), burying the handful of real changes in +// reformatting noise and making the nightly PR unreviewable. Even re-encoding +// just the candidates subtree would restyle authored fields such as min_vram, +// so the rewrite reaches down to individual mapping keys and the node tree is +// re-encoded at the index's authored indent. That also makes the job +// idempotent: a run that computes the same values leaves the file untouched +// and opens no PR. func writeCandidates(path string, data []byte, entries []gallery.GalleryModel) error { var doc yaml.Node if err := yaml.Unmarshal(data, &doc); err != nil { @@ -180,17 +185,50 @@ func writeCandidates(path string, data []byte, entries []gallery.GalleryModel) e } } - // Re-encoding an untouched document still normalizes indentation, so skip - // the write entirely when there was nothing to rewrite. + // Nothing to rewrite means nothing to encode: leave the file, and its + // mtime, alone. if !changed { return nil } - out, err := yaml.Marshal(&doc) + // yaml.Marshal encodes at yaml.v3's default 4-space indent, which reflows + // every nested block in the file (~6000 lines against the real index) and + // drowns the handful of rewritten keys. The index is authored at 2, so + // encode at 2 and the diff stays limited to what actually changed. + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + if err := enc.Encode(&doc); err != nil { + return fmt.Errorf("encode: %w", err) + } + if err := enc.Close(); err != nil { + return fmt.Errorf("flush encoder: %w", err) + } + + // The encoder does not emit a document start marker, so restore the one the + // file was authored with rather than deleting it on every run. + out := buf.Bytes() + if header := documentHeader(data); header != nil && !bytes.HasPrefix(out, header) { + out = append(header, out...) + } + + // Preserve the file's existing mode instead of forcing 0644. + info, err := os.Stat(path) if err != nil { - return fmt.Errorf("marshal: %w", err) + return fmt.Errorf("stat: %w", err) } - return os.WriteFile(path, out, 0644) + return os.WriteFile(path, out, info.Mode().Perm()) +} + +// documentHeader returns the leading document start marker line, or nil when the +// file was authored without one. +func documentHeader(data []byte) []byte { + for _, marker := range [][]byte{[]byte("---\n"), []byte("---\r\n")} { + if bytes.HasPrefix(data, marker) { + return marker + } + } + return nil } // mappingValue returns the value node for key, or nil when absent. diff --git a/.github/workflows/gallery-denormalize.yaml b/.github/workflows/gallery-denormalize.yaml index 14551d3b1044..e58a83e1f949 100644 --- a/.github/workflows/gallery-denormalize.yaml +++ b/.github/workflows/gallery-denormalize.yaml @@ -12,8 +12,25 @@ jobs: - uses: actions/setup-go@v5 with: go-version-file: go.mod + # The program exits 1 when any candidate could not be estimated, but it + # still writes everything it did compute. Aborting here would let one + # unreachable candidate block the refresh of every other one, night after + # night, so record the status and let the PR open with the partial result. - name: Denormalize meta candidates - run: go run ./.github/ci/gallery_denormalize.go gallery/index.yaml + id: denormalize + run: | + set +e + go run ./.github/ci/gallery_denormalize.go gallery/index.yaml + status=$? + set -e + echo "status=$status" >> "$GITHUB_OUTPUT" + if [ "$status" -ne 0 ]; then + echo "partial=yes" >> "$GITHUB_OUTPUT" + echo "note=**This run was partial.** At least one candidate could not be estimated, so some fields may be missing or stale. See the workflow log." >> "$GITHUB_OUTPUT" + else + echo "partial=no" >> "$GITHUB_OUTPUT" + echo "note=All candidates were processed successfully." >> "$GITHUB_OUTPUT" + fi - name: Create Pull Request uses: peter-evans/create-pull-request@v8 with: @@ -27,4 +44,13 @@ jobs: `inferred_min_vram` fields on meta model entries. Authored `min_vram` values are never modified. + + ${{ steps.denormalize.outputs.note }} signoff: true + # Surface the failure only after the PR exists, so the problem stays + # visible without discarding the work that did succeed. + - name: Report estimation failures + if: steps.denormalize.outputs.partial == 'yes' + run: | + echo "gallery_denormalize exited ${{ steps.denormalize.outputs.status }}: one or more candidates need a hand-authored min_vram." + exit 1 From d0d441bb435f104f5ae130de19621894fb2150fa Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 11:41:16 +0000 Subject: [PATCH 14/48] feat(gallery): add nanbeige4.1-3b meta entry with hardware-resolved variants Adds the first real meta entry to the gallery index. It resolves to the Q8_0 build on hosts with at least 6GiB of VRAM and to the Q4_K_M build everywhere else, installing either payload under the stable name nanbeige4.1-3b. The entry carries a url equal to its final candidate's url. LocalAI releases that predate candidates support parse the index non-strictly and drop the key silently, so without that url they would list the entry and install nothing. A regression spec parses the index the way those releases do and asserts every meta entry stays installable for them. Also teaches core/schema/gallery-model.schema.json about candidates. The schema sets additionalProperties: false at the top level, so an author following CONTRIBUTING.md and adding the yaml-language-server comment would otherwise get a validation error on this entry. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/gallery/meta_install_test.go | 37 +++++++++++++++++++++++++++ core/schema/gallery-model.schema.json | 36 ++++++++++++++++++++++++++ gallery/index.yaml | 29 +++++++++++++++++++++ 3 files changed, 102 insertions(+) diff --git a/core/gallery/meta_install_test.go b/core/gallery/meta_install_test.go index df730b3cb21c..ee57162a7445 100644 --- a/core/gallery/meta_install_test.go +++ b/core/gallery/meta_install_test.go @@ -368,3 +368,40 @@ var _ = Describe("InstallModelFromGallery with meta entries", func() { Expect(record.URLs).To(ConsistOf("https://example.invalid/qwen3")) }) }) + +var _ = Describe("legacy client compatibility", func() { + It("exposes a url on every meta entry to clients that ignore candidates", func() { + data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) + Expect(err).ToNot(HaveOccurred()) + + // Parse exactly as an older LocalAI release would: non-strictly, with + // no knowledge of the candidates key. + var legacy []struct { + Name string `yaml:"name"` + URL string `yaml:"url"` + } + Expect(yaml.Unmarshal(data, &legacy)).To(Succeed()) + + var current []gallery.GalleryModel + Expect(yaml.Unmarshal(data, ¤t)).To(Succeed()) + + urlByName := map[string]string{} + for _, e := range legacy { + urlByName[e.Name] = e.URL + } + + metaCount := 0 + for _, e := range current { + if !e.IsMeta() { + continue + } + metaCount++ + // Without a url an old client lists the entry and installs an + // empty model, because it silently drops candidates. + Expect(urlByName[e.Name]).ToNot(BeEmpty(), + "meta entry %q is invisible payload-wise to older clients", e.Name) + } + Expect(metaCount).To(BeNumerically(">", 0), + "expected at least one meta entry in the gallery index") + }) +}) diff --git a/core/schema/gallery-model.schema.json b/core/schema/gallery-model.schema.json index 9ce13ac0f6ec..d43b42e07ad5 100644 --- a/core/schema/gallery-model.schema.json +++ b/core/schema/gallery-model.schema.json @@ -56,6 +56,42 @@ "additionalProperties": false } }, + "candidates": { + "type": "array", + "description": "Ordered variant list of a meta entry. The first candidate the host satisfies is installed, under the meta entry's own name. An entry carrying candidates must not also carry files or config_file.", + "minItems": 1, + "items": { + "type": "object", + "required": ["model"], + "properties": { + "model": { + "type": "string", + "description": "Name of a concrete (non-meta) gallery entry" + }, + "capability": { + "type": "string", + "description": "Host capability this candidate requires, matched exactly and case-sensitively (for example metal, nvidia-cuda-12). Absent matches any host." + }, + "min_vram": { + "type": "string", + "description": "Authored VRAM floor, for example 20GiB. Authoritative: inference never overwrites it." + }, + "backend": { + "type": "string", + "description": "Denormalized by the nightly job for display. Never authored by hand." + }, + "quantization": { + "type": "string", + "description": "Denormalized by the nightly job for display. Never authored by hand." + }, + "inferred_min_vram": { + "type": "string", + "description": "Denormalized by the nightly job as a fallback floor. Never authored by hand; min_vram wins." + } + }, + "additionalProperties": false + } + }, "prompt_templates": { "type": "array", "description": "Prompt templates written as .tmpl files", diff --git a/gallery/index.yaml b/gallery/index.yaml index 79e7e265a49d..c449afa6e445 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -4234,6 +4234,35 @@ - filename: llama-cpp/models/Qwen_Qwen3-Next-80B-A3B-Thinking-Q4_K_M.gguf sha256: 83481c75cc6c0837ba9afa52b59b4cd3f85f55dd7aa6c60e27230ff329c81367 uri: https://huggingface.co/bartowski/Qwen_Qwen3-Next-80B-A3B-Thinking-GGUF/resolve/main/Qwen_Qwen3-Next-80B-A3B-Thinking-Q4_K_M.gguf +- name: nanbeige4.1-3b + description: | + Nanbeige4.1-3B is built upon Nanbeige4-3B-Base and represents an enhanced iteration of our previous reasoning model, Nanbeige4-3B-Thinking-2511, achieved through further post-training optimization with supervised fine-tuning (SFT) and reinforcement learning (RL). As a highly competitive open-source model at a small parameter scale, Nanbeige4.1-3B illustrates that compact models can simultaneously achieve robust reasoning, preference alignment, and effective agentic behaviors. + + Automatically resolves to the best variant for your hardware: the Q8_0 build where there is enough VRAM for it, the Q4_K_M build otherwise. + license: apache-2.0 + icon: https://cdn-avatars.huggingface.co/v1/production/uploads/646f0d118ff94af23bc44aab/GXHCollpMRgvYqUXQ2BQ7.png + urls: + - https://huggingface.co/Nanbeige/Nanbeige4.1-3B + tags: + - nanbeige + - 3b + - llm + - gguf + - quantized + - chat + - reasoning + - agent + - multilingual + - instruction-tuned + # url is the fallback for LocalAI releases that predate candidates support: + # they drop the candidates key silently and would otherwise install nothing. + # Lint requires it to equal the final candidate's url exactly, so old and new + # clients agree on the least demanding option. + url: github:mudler/LocalAI/gallery/nanbeige4.1.yaml@master + candidates: + - model: nanbeige4.1-3b-q8 + min_vram: 6GiB + - model: nanbeige4.1-3b-q4 - name: nanbeige4.1-3b-q8 url: github:mudler/LocalAI/gallery/nanbeige4.1.yaml@master urls: From 30b307293a864cefa75553a19b6208503ca1fba9 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 21:32:12 +0000 Subject: [PATCH 15/48] feat(gallery): make candidate entries complete, installable entries Reworks hardware-resolved gallery variants after a design pivot. There is no longer a separate "meta" entry kind. A gallery entry is a normal, complete entry that may additionally carry candidates:, a list of hardware-gated upgrades over itself, and the entry is itself the last-resort candidate. The previous design relied on a bare url: as the fallback for LocalAI releases that predate candidates support. That fallback is empty in practice: none of the 80 gallery/*.yaml files carry a top-level files:, and 1216 of 1281 index entries carry their payload in the index entry itself, so a url alone yields a config template with nothing to download. Since every released LocalAI reads gallery/index.yaml live from master, merging a payload-less entry would have shown every existing user a model that installs to a broken state. Making the entry its own base candidate removes the problem at the root: old clients drop the candidates key and install the entry exactly as they do today. Resolution order is now explicit pin, then capability plus VRAM over the declared upgrades, then the entry itself. The entry ALWAYS installs: when its own min_vram or capability is unmet the installer warns and installs it anyway, because there is nothing below it and refusing would make the gallery behave worse the newer the client is. A pin naming the entry's own name is valid and is how an operator declines an upgrade. IsMeta() becomes HasCandidates(), ResolveMetaModel becomes ResolveVariant, and the persisted meta_name record key becomes entry_name. GalleryBackend.IsMeta() is a separate concept and is untouched. The lint drops the three rules the pivot makes wrong (url equality with the final candidate, no inline payload, unconstrained final candidate) and gains one: the entry's own floor must sit strictly below every candidate's, since a base that outranks a candidate makes that candidate unreachable. The pilot entry is now the existing nanbeige4.1-3b-q4, which gains a 2GiB floor of its own and a single 6GiB upgrade to nanbeige4.1-3b-q8, replacing the separate nanbeige4.1-3b entry added in d0d441bb4. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- .github/ci/gallery_denormalize.go | 9 +- core/gallery/candidate.go | 12 +- core/gallery/candidates_install_test.go | 480 +++++++++++++++++++++ core/gallery/candidates_lint_test.go | 544 ++++++++++++++++++++++++ core/gallery/meta_install_test.go | 407 ------------------ core/gallery/meta_lint_test.go | 510 ---------------------- core/gallery/model_artifacts.go | 6 +- core/gallery/models.go | 134 ++++-- core/gallery/models_types.go | 23 +- core/schema/gallery-model.schema.json | 12 +- gallery/index.yaml | 37 +- 11 files changed, 1164 insertions(+), 1010 deletions(-) create mode 100644 core/gallery/candidates_install_test.go create mode 100644 core/gallery/candidates_lint_test.go delete mode 100644 core/gallery/meta_install_test.go delete mode 100644 core/gallery/meta_lint_test.go diff --git a/.github/ci/gallery_denormalize.go b/.github/ci/gallery_denormalize.go index a650a64580e9..0ae528f4705e 100644 --- a/.github/ci/gallery_denormalize.go +++ b/.github/ci/gallery_denormalize.go @@ -1,5 +1,6 @@ -// Command gallery_denormalize fills the read-only denormalized fields on meta -// model entries: backend, quantization, and inferred_min_vram. +// Command gallery_denormalize fills the read-only denormalized fields on the +// candidates of gallery model entries: backend, quantization, and +// inferred_min_vram. // // It never modifies an authored min_vram. Authored values are authoritative, // because a human who measured a real load knows more than a pre-download @@ -50,7 +51,7 @@ func main() { failures := 0 for i := range entries { - if !entries[i].IsMeta() { + if !entries[i].HasCandidates() { continue } for j := range entries[i].Candidates { @@ -155,7 +156,7 @@ func writeCandidates(path string, data []byte, entries []gallery.GalleryModel) e changed := false for i, entryNode := range root.Content { - if !entries[i].IsMeta() || entryNode.Kind != yaml.MappingNode { + if !entries[i].HasCandidates() || entryNode.Kind != yaml.MappingNode { continue } diff --git a/core/gallery/candidate.go b/core/gallery/candidate.go index cede3d934f17..9e5b342981a3 100644 --- a/core/gallery/candidate.go +++ b/core/gallery/candidate.go @@ -6,11 +6,13 @@ import ( "github.com/mudler/LocalAI/pkg/vram" ) -// Candidate is one option in a meta model entry's ordered candidate list. It -// references an existing concrete gallery entry by name and declares the -// conditions under which that entry is the right choice for the host. +// Candidate is one option in a gallery entry's ordered candidate list. It +// references an existing gallery entry by name and declares the conditions +// under which that entry is the right choice for the host. type Candidate struct { - // Model is the name of a concrete (non-meta) gallery entry. + // Model is the name of a gallery entry that declares no candidates of its + // own, or the name of the declaring entry itself when it stands as its own + // last-resort candidate. Model string `json:"model" yaml:"model"` // Capability, when set, must equal the host's reported capability // (e.g. "metal", "nvidia-cuda-12"). Empty matches any host. @@ -21,7 +23,7 @@ type Candidate struct { // The fields below are denormalized by the nightly job for display and // lint. They are never authored by hand and never affect what gets - // installed, because installation reads the concrete entry live. + // installed, because installation reads the referenced entry live. Backend string `json:"backend,omitempty" yaml:"backend,omitempty"` Quantization string `json:"quantization,omitempty" yaml:"quantization,omitempty"` InferredMinVRAM string `json:"inferred_min_vram,omitempty" yaml:"inferred_min_vram,omitempty"` diff --git a/core/gallery/candidates_install_test.go b/core/gallery/candidates_install_test.go new file mode 100644 index 000000000000..108f26e6605b --- /dev/null +++ b/core/gallery/candidates_install_test.go @@ -0,0 +1,480 @@ +package gallery_test + +import ( + "context" + "os" + "path/filepath" + + "dario.cat/mergo" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/system" +) + +var _ = Describe("GalleryModel candidate declarations", func() { + It("declares no candidates when the key is absent", func() { + m := gallery.GalleryModel{} + m.Name = "plain" + Expect(m.HasCandidates()).To(BeFalse()) + }) + + It("declares candidates when the key is present", func() { + m := gallery.GalleryModel{Candidates: []gallery.Candidate{{Model: "x"}}} + Expect(m.HasCandidates()).To(BeTrue()) + }) + + It("parses an entry's own selection fields and its candidate list in order", func() { + var m gallery.GalleryModel + err := yaml.Unmarshal([]byte(` +name: qwen3-8b-gguf-q4 +url: "github:example/repo/qwen3-8b-gguf-q4.yaml@master" +min_vram: 6GiB +capability: default +candidates: + - model: qwen3-8b-vllm-awq + capability: nvidia + min_vram: 20GiB + - model: qwen3-8b-gguf-q8 + min_vram: 10GiB +`), &m) + Expect(err).ToNot(HaveOccurred()) + Expect(m.Name).To(Equal("qwen3-8b-gguf-q4")) + Expect(m.URL).To(Equal("github:example/repo/qwen3-8b-gguf-q4.yaml@master")) + Expect(m.MinVRAM).To(Equal("6GiB")) + Expect(m.Capability).To(Equal("default")) + Expect(m.HasCandidates()).To(BeTrue()) + Expect(m.Candidates).To(HaveLen(2)) + Expect(m.Candidates[0].Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(m.Candidates[0].Capability).To(Equal("nvidia")) + Expect(m.Candidates[0].MinVRAM).To(Equal("20GiB")) + Expect(m.Candidates[1].Model).To(Equal("qwen3-8b-gguf-q8")) + Expect(m.Candidates[1].Capability).To(BeEmpty()) + }) +}) + +var _ = Describe("ResolveVariant", func() { + gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 } + + newModel := func(name, url, description, icon string) *gallery.GalleryModel { + m := &gallery.GalleryModel{} + m.Name = name + m.URL = url + m.Description = description + m.Icon = icon + return m + } + + var models []*gallery.GalleryModel + var base *gallery.GalleryModel + + BeforeEach(func() { + upgrade := newModel("qwen3-8b-vllm-awq", "file://vllm.yaml", "AWQ variant", "vllm.png") + // The base is an ordinary, complete entry that happens to declare an + // upgrade over itself, which is the whole point of the design. + base = newModel("qwen3-8b-gguf-q4", "file://gguf.yaml", "Qwen3 8B Q4", "qwen.png") + base.Tags = []string{"llm"} + base.MinVRAM = "6GiB" + base.Candidates = []gallery.Candidate{ + {Model: "qwen3-8b-vllm-awq", Capability: "nvidia", MinVRAM: "20GiB"}, + } + models = []*gallery.GalleryModel{upgrade, base} + }) + + It("installs a matching candidate's payload under the entry's name", func() { + resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(candidate.Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(resolved.Name).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://vllm.yaml")) + }) + + It("falls back to the entry's own payload when no candidate fits", func() { + resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) + }) + + It("installs the entry even when the host misses the entry's own floor", func() { + // There is nothing below the base, so refusing here would make an entry + // that every older client installs fine uninstallable on new ones. + resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(1)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) + }) + + It("strips the selection fields from the resolved entry", func() { + // A resolved entry is a concrete install target. Leaving the fields on + // it would let a second resolution pass fire on an already-resolved + // entry. + resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(resolved.HasCandidates()).To(BeFalse()) + Expect(resolved.MinVRAM).To(BeEmpty()) + Expect(resolved.Capability).To(BeEmpty()) + }) + + It("presents the entry's metadata, not the candidate's", func() { + resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(resolved.Description).To(Equal("Qwen3 8B Q4")) + Expect(resolved.Icon).To(Equal("qwen.png")) + Expect(resolved.Tags).To(ConsistOf("llm")) + }) + + It("honors a pin naming the entry itself", func() { + // The entry is the last element of its own candidate list, so its own + // name has to be a usable pin: it is how an operator declines an + // upgrade their hardware would otherwise take. + resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "qwen3-8b-gguf-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.Name).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) + }) + + It("honors a pin the hardware does not satisfy", func() { + resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(2)}, "qwen3-8b-vllm-awq") + Expect(err).ToNot(HaveOccurred()) + Expect(candidate.Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(resolved.URL).To(Equal("file://vllm.yaml")) + }) + + It("detaches the resolved entry from the gallery's own maps and slices", func() { + models[0].Overrides = map[string]any{"f16": true} + models[0].AdditionalFiles = []gallery.File{{Filename: "a.bin"}} + + resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + + // The install path merges the caller's request overrides into this map in + // place, so aliasing the gallery's map would write one caller's request + // into the shared catalog and leak it into every later install. + resolved.Overrides["f16"] = false + resolved.Overrides["threads"] = 4 + Expect(models[0].Overrides).To(HaveKeyWithValue("f16", true)) + Expect(models[0].Overrides).ToNot(HaveKey("threads")) + + resolved.AdditionalFiles[0].Filename = "mutated.bin" + Expect(models[0].AdditionalFiles[0].Filename).To(Equal("a.bin")) + + resolved.Tags = append(resolved.Tags, "extra") + Expect(base.Tags).To(ConsistOf("llm")) + }) + + It("detaches the resolved entry even when it resolves to the entry itself", func() { + // Resolving to the base returns a copy of the very entry the gallery + // holds, which is the case most likely to alias it. + base.Overrides = map[string]any{"parameters": map[string]any{"model": "q4.gguf"}} + + resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).ToNot(HaveOccurred()) + + resolved.Overrides["parameters"].(map[string]any)["model"] = "callers-choice.gguf" + Expect(base.Overrides["parameters"]).To(HaveKeyWithValue("model", "q4.gguf")) + }) + + It("detaches nested override maps from the gallery's own entry", func() { + models[0].Overrides = map[string]any{ + "parameters": map[string]any{"model": "real-variant.gguf"}, + "stopwords": []any{""}, + } + + resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + + // Cloning only the top level would leave this inner map shared with the + // gallery entry, so writing through the resolved copy would rewrite the + // catalog's own payload. + resolved.Overrides["parameters"].(map[string]any)["model"] = "callers-choice.gguf" + resolved.Overrides["stopwords"].([]any)[0] = "<|im_end|>" + + Expect(models[0].Overrides["parameters"]).To(HaveKeyWithValue("model", "real-variant.gguf")) + Expect(models[0].Overrides["stopwords"]).To(Equal([]any{""})) + }) + + It("does not write the caller's overrides back into the gallery entry", func() { + models[0].Overrides = map[string]any{"parameters": map[string]any{"model": "real-variant.gguf"}} + + resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + Expect(err).ToNot(HaveOccurred()) + + // This is exactly what the install path does with the caller's request + // overrides, and mergo recurses into nested maps and overwrites them in + // place. Asserting against the in-memory catalog is the only way to + // observe the leak: re-reading the gallery from disk re-unmarshals fresh + // maps and would pass whether or not the resolved entry was detached. + requestOverrides := map[string]any{"parameters": map[string]any{"model": "callers-choice.gguf"}} + Expect(mergo.Merge(&resolved.Overrides, requestOverrides, mergo.WithOverride)).To(Succeed()) + + Expect(resolved.Overrides["parameters"]).To(HaveKeyWithValue("model", "callers-choice.gguf")) + Expect(models[0].Overrides["parameters"]).To(HaveKeyWithValue("model", "real-variant.gguf")) + }) + + It("errors when a candidate references a missing entry", func() { + base.Candidates = []gallery.Candidate{{Model: "does-not-exist"}} + _, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does-not-exist")) + }) + + It("refuses a candidate that declares candidates of its own", func() { + nested := newModel("nested", "file://nested.yaml", "", "") + nested.Candidates = []gallery.Candidate{{Model: "qwen3-8b-vllm-awq"}} + models = append(models, nested) + base.Candidates = []gallery.Candidate{{Model: "nested"}} + _, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("nested")) + }) + + It("surfaces a bad pin", func() { + _, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "nope") + Expect(err).To(MatchError(gallery.ErrPinNotFound)) + }) +}) + +var _ = Describe("InstallModelFromGallery with candidate entries", func() { + var tempdir string + var galleries []config.Gallery + var systemState *system.SystemState + + // Every entry is described with an inline config_file rather than a URL so + // the whole install runs off the local filesystem with no network access. + newGallery := func(entries ...gallery.GalleryModel) { + out, err := yaml.Marshal(entries) + Expect(err).ToNot(HaveOccurred()) + galleryPath := filepath.Join(tempdir, "gallery.yaml") + Expect(os.WriteFile(galleryPath, out, 0600)).To(Succeed()) + + galleries = []config.Gallery{{Name: "test", URL: "file://" + galleryPath}} + } + + entry := func(name, backend string) gallery.GalleryModel { + m := gallery.GalleryModel{ConfigFile: map[string]any{"backend": backend}} + m.Name = name + m.Description = "entry " + name + return m + } + + // urlEntry describes an entry through a url rather than an inline + // config_file. The distinction matters for the recorded name: the url branch + // reads a name out of the fetched config, and that name is the referenced + // entry's own, so only the entry-name overlay can keep the record under the + // name the user asked for. The inline config_file branch seeds the name from + // the already-renamed resolved entry and so cannot observe the overlay. + urlEntry := func(name, backend string) gallery.GalleryModel { + payload := gallery.ModelConfig{ + Name: name, + Description: "entry " + name, + ConfigFile: "backend: " + backend + "\n", + } + out, err := yaml.Marshal(payload) + Expect(err).ToNot(HaveOccurred()) + payloadPath := filepath.Join(tempdir, "payload-"+name+".yaml") + Expect(os.WriteFile(payloadPath, out, 0600)).To(Succeed()) + + m := gallery.GalleryModel{} + m.Name = name + m.Description = "entry " + name + m.URL = "file://" + payloadPath + return m + } + + // withCandidates attaches upgrades to an otherwise ordinary entry. The + // floors are absolute rather than relative to the host: "0GiB" always + // matches and "10000GiB" never does, so these specs assert on resolution + // rather than on whatever VRAM the machine running them happens to have. + withCandidates := func(m gallery.GalleryModel, candidates ...gallery.Candidate) gallery.GalleryModel { + m.Candidates = candidates + return m + } + + install := func(name string, req gallery.GalleryModel, options ...gallery.InstallOption) error { + return gallery.InstallModelFromGallery( + context.TODO(), galleries, []config.Gallery{}, systemState, nil, + name, req, func(string, string, string, float64) {}, false, false, false, options...) + } + + installedBackend := func(name string) string { + dat, err := os.ReadFile(filepath.Join(tempdir, name+".yaml")) + Expect(err).ToNot(HaveOccurred()) + content := map[string]any{} + Expect(yaml.Unmarshal(dat, &content)).To(Succeed()) + return content["backend"].(string) + } + + BeforeEach(func() { + var err error + tempdir, err = os.MkdirTemp("", "candidate-install") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { Expect(os.RemoveAll(tempdir)).To(Succeed()) }) + + systemState, err = system.GetSystemState(system.WithModelPath(tempdir)) + Expect(err).ToNot(HaveOccurred()) + }) + + It("installs the entry's own payload when no candidate fits the host", func() { + newGallery( + withCandidates(entry("qwen3-8b-q4", "base-backend"), + gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "10000GiB"}), + entry("qwen3-8b-q8", "upgrade-backend"), + ) + + // No machine clears a 10000GiB floor, so this asserts the base is the + // last resort and that missing every candidate is not an error. + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend")) + }) + + It("installs a fitting candidate's payload under the entry's own name", func() { + newGallery( + withCandidates(entry("qwen3-8b-q4", "base-backend"), + gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}), + entry("qwen3-8b-q8", "upgrade-backend"), + ) + + // A 0GiB floor is met by every machine, so this asserts the upgrade wins + // over the base and lands under the base's name rather than its own. + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("upgrade-backend")) + _, err := os.Stat(filepath.Join(tempdir, "qwen3-8b-q8.yaml")) + Expect(os.IsNotExist(err)).To(BeTrue(), "the upgrade must not be installed under its own name") + }) + + It("round-trips the resolution record to disk under the entry's name", func() { + // The upgrade is described by url so its payload carries its own name. + // Without the entry-name overlay the record persists as "qwen3-8b-q8", + // and the stable name the entry exists to provide is lost the moment + // anything reads the record back. + newGallery( + withCandidates(urlEntry("qwen3-8b-q4", "base-backend"), + gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}), + urlEntry("qwen3-8b-q8", "upgrade-backend"), + ) + + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + + record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(record.EntryName).To(Equal("qwen3-8b-q4")) + Expect(record.ResolvedVariant).To(Equal("qwen3-8b-q8")) + Expect(record.PinnedVariant).To(BeEmpty()) + Expect(record.Name).To(Equal("qwen3-8b-q4")) + Expect(record.Description).To(Equal("entry qwen3-8b-q4")) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("upgrade-backend")) + }) + + It("records a pin and honors it on a plain reinstall", func() { + newGallery( + withCandidates(entry("qwen3-8b-q4", "base-backend"), + gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}), + entry("qwen3-8b-q8", "upgrade-backend"), + ) + + Expect(install("qwen3-8b-q4", gallery.GalleryModel{}, gallery.WithVariant("qwen3-8b-q4"))).To(Succeed()) + + record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(record.PinnedVariant).To(Equal("qwen3-8b-q4")) + Expect(record.ResolvedVariant).To(Equal("qwen3-8b-q4")) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend")) + + // No WithVariant this time: hardware resolution would take the upgrade, + // so only the recalled pin can keep this on the base payload. + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend")) + + record, err = gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(record.PinnedVariant).To(Equal("qwen3-8b-q4")) + }) + + It("honors a pin recorded under a custom install name", func() { + newGallery( + withCandidates(entry("qwen3-8b-q4", "base-backend"), + gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}), + entry("qwen3-8b-q8", "upgrade-backend"), + ) + + req := gallery.GalleryModel{} + req.Name = "prod-llm" + + Expect(install("qwen3-8b-q4", req, gallery.WithVariant("qwen3-8b-q4"))).To(Succeed()) + + // The pin is written under the installed name, so the recall must read it + // back under that name too and not under the gallery entry's name. + record, err := gallery.GetLocalModelConfiguration(tempdir, "prod-llm") + Expect(err).ToNot(HaveOccurred()) + Expect(record.PinnedVariant).To(Equal("qwen3-8b-q4")) + Expect(record.EntryName).To(Equal("qwen3-8b-q4")) + Expect(installedBackend("prod-llm")).To(Equal("base-backend")) + + Expect(install("qwen3-8b-q4", req)).To(Succeed()) + Expect(installedBackend("prod-llm")).To(Equal("base-backend")) + }) + + It("writes each declared url only once into the persisted gallery file", func() { + base := withCandidates(entry("qwen3-8b-q4", "base-backend"), + gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}) + base.URLs = []string{"https://example.invalid/qwen3"} + newGallery(base, entry("qwen3-8b-q8", "upgrade-backend")) + + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + + record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(record.URLs).To(ConsistOf("https://example.invalid/qwen3")) + }) +}) + +var _ = Describe("legacy client compatibility", func() { + It("keeps every entry that declares candidates installable by clients that ignore them", func() { + data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) + Expect(err).ToNot(HaveOccurred()) + + // Parse exactly as an older LocalAI release would: non-strictly, with + // no knowledge of the candidates key. Such a client installs whatever + // payload the entry carries directly, so the entry must carry one. + var legacy []struct { + Name string `yaml:"name"` + URL string `yaml:"url"` + ConfigFile map[string]any `yaml:"config_file"` + Files []gallery.File `yaml:"files"` + Overrides map[string]any `yaml:"overrides"` + } + Expect(yaml.Unmarshal(data, &legacy)).To(Succeed()) + + var current []gallery.GalleryModel + Expect(yaml.Unmarshal(data, ¤t)).To(Succeed()) + + legacyByName := map[string]int{} + for i, e := range legacy { + legacyByName[e.Name] = i + } + + withCandidates := 0 + for _, e := range current { + if !e.HasCandidates() { + continue + } + withCandidates++ + + i, ok := legacyByName[e.Name] + Expect(ok).To(BeTrue(), "entry %q vanished under a legacy parse", e.Name) + old := legacy[i] + // An entry whose payload lived only in its candidates would install + // to nothing on every released LocalAI, which is precisely what + // making the entry itself the base candidate exists to prevent. + Expect(old.URL != "" || len(old.ConfigFile) > 0).To(BeTrue(), + "entry %q carries no payload of its own, so older clients would install nothing", e.Name) + } + Expect(withCandidates).To(BeNumerically(">", 0), + "expected at least one entry declaring candidates in the gallery index") + }) +}) diff --git a/core/gallery/candidates_lint_test.go b/core/gallery/candidates_lint_test.go new file mode 100644 index 000000000000..5ba9dd52d4a5 --- /dev/null +++ b/core/gallery/candidates_lint_test.go @@ -0,0 +1,544 @@ +package gallery_test + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/mudler/LocalAI/core/gallery" +) + +// knownCapabilities mirrors the values SystemState.DetectedCapability() can +// actually report, which is what the candidate resolver compares against with +// a case-sensitive exact match. A capability outside this set can never match, +// so a typo would silently make a candidate unreachable on every host. +// +// Note "cpu" is deliberately absent: it exists only as a fallback key inside +// SystemState.Capability(capMap) for meta backends, and is never a value +// getSystemCapabilities() returns. A CPU-only host reports "default". +var knownCapabilities = map[string]bool{ + "default": true, "metal": true, "darwin-x86": true, + "nvidia": true, "nvidia-cuda-12": true, "nvidia-cuda-13": true, + "nvidia-l4t": true, "nvidia-l4t-cuda-12": true, "nvidia-l4t-cuda-13": true, + "intel": true, "amd": true, "vulkan": true, +} + +// candidateViolation is one invariant breach found by a lint helper. Helpers +// return every breach they find instead of stopping at the first, so a single +// run names them all rather than forcing a fix-one-rerun-repeat cycle. +type candidateViolation struct { + Entry string + Candidate string + Detail string +} + +func (v candidateViolation) String() string { + if v.Candidate == "" { + return fmt.Sprintf("%s: %s", v.Entry, v.Detail) + } + return fmt.Sprintf("%s -> candidate %q: %s", v.Entry, v.Candidate, v.Detail) +} + +func formatViolations(violations []candidateViolation) string { + lines := make([]string, 0, len(violations)) + for _, v := range violations { + lines = append(lines, " "+v.String()) + } + return "\n" + strings.Join(lines, "\n") +} + +func indexEntriesByName(entries []gallery.GalleryModel) map[string]gallery.GalleryModel { + byName := make(map[string]gallery.GalleryModel, len(entries)) + for _, e := range entries { + byName[e.Name] = e + } + return byName +} + +// baseCandidate mirrors the implicit last-resort candidate the resolver +// synthesizes from the entry itself, so lint measures the same floor the +// installer will. +func baseCandidate(e gallery.GalleryModel) gallery.Candidate { + return gallery.Candidate{Model: e.Name, Capability: e.Capability, MinVRAM: e.MinVRAM} +} + +// checkCandidateReferences verifies every candidate names an existing entry +// that declares no candidates of its own. Resolution is a single pass, so a +// nested reference would silently ignore the inner list. +func checkCandidateReferences(entries []gallery.GalleryModel) []candidateViolation { + byName := indexEntriesByName(entries) + var violations []candidateViolation + for _, e := range entries { + if !e.HasCandidates() { + continue + } + for _, c := range e.Candidates { + target, ok := byName[c.Model] + if !ok { + violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "references unknown model"}) + continue + } + if target.HasCandidates() { + violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "references an entry that declares candidates of its own; nesting is not allowed"}) + } + } + } + return violations +} + +// checkCandidateFloors verifies every declared candidate carries a parseable +// VRAM floor. A candidate is an UPGRADE over the entry, so it must say what it +// costs; a floorless one would capture every host and make the entry's own +// payload unreachable. +func checkCandidateFloors(entries []gallery.GalleryModel) []candidateViolation { + var violations []candidateViolation + for _, e := range entries { + if !e.HasCandidates() { + continue + } + for _, c := range e.Candidates { + _, declared, err := c.EffectiveMinVRAM() + switch { + case err != nil: + violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "has a bad min_vram: " + err.Error()}) + case !declared: + violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "needs a min_vram; the nightly job should have inferred one"}) + } + } + } + return violations +} + +// checkBaseFloor verifies the entry's own floor sits strictly below every +// candidate's. The entry is the last element of its own candidate list, so a +// base floor at or above a candidate's makes that candidate dead: any host +// clearing it already cleared the base, and first-match never gets that far. +func checkBaseFloor(entries []gallery.GalleryModel) []candidateViolation { + var violations []candidateViolation + for _, e := range entries { + if !e.HasCandidates() { + continue + } + base := baseCandidate(e) + baseFloor, _, err := base.EffectiveMinVRAM() + if err != nil { + violations = append(violations, candidateViolation{Entry: e.Name, Detail: "has a bad min_vram: " + err.Error()}) + continue + } + for _, c := range e.Candidates { + floor, declared, cerr := c.EffectiveMinVRAM() + if cerr != nil || !declared { + // checkCandidateFloors owns reporting those. + continue + } + if baseFloor >= floor { + violations = append(violations, candidateViolation{ + Entry: e.Name, + Candidate: c.Model, + Detail: fmt.Sprintf("sits at or below the entry's own %d byte floor, so it can never be reached", + baseFloor), + }) + } + } + } + return violations +} + +// checkCandidateCapabilities verifies entries and candidates only name +// capabilities the system can actually report, since the resolver compares +// them exactly. +func checkCandidateCapabilities(entries []gallery.GalleryModel) []candidateViolation { + var violations []candidateViolation + for _, e := range entries { + if !e.HasCandidates() { + continue + } + if e.Capability != "" && !knownCapabilities[e.Capability] { + violations = append(violations, candidateViolation{ + Entry: e.Name, + Detail: fmt.Sprintf("uses unknown capability %q", e.Capability), + }) + } + for _, c := range e.Candidates { + if c.Capability == "" { + continue + } + if !knownCapabilities[c.Capability] { + violations = append(violations, candidateViolation{ + Entry: e.Name, + Candidate: c.Model, + Detail: fmt.Sprintf("uses unknown capability %q", c.Capability), + }) + } + } + } + return violations +} + +// checkCandidateOrdering verifies no candidate is shadowed by an earlier one. +// Selection is first-match over the authored order, so a candidate is dead +// whenever an earlier one matches every host this one would. +// +// Two shapes cause that. Within a single capability group the groups are +// mutually exclusive, so only a floor rising above an earlier floor is +// unreachable. A candidate with an EMPTY capability matches every host and so +// dominates ACROSS groups: every later candidate whose floor is at or above +// the running minimum unconditional floor is dead, whatever capability it asks +// for. Tracking that running minimum subsumes the same-group check for the +// empty capability. +func checkCandidateOrdering(entries []gallery.GalleryModel) []candidateViolation { + var violations []candidateViolation + for _, e := range entries { + if !e.HasCandidates() { + continue + } + previous := map[string]uint64{} + var unconditionalFloor uint64 + haveUnconditional := false + + for _, c := range e.Candidates { + floor, declared, err := c.EffectiveMinVRAM() + if err != nil || !declared { + // Parse errors and absent floors belong to checkCandidateFloors. + continue + } + + switch { + case haveUnconditional && floor >= unconditionalFloor: + violations = append(violations, candidateViolation{ + Entry: e.Name, + Candidate: c.Model, + Detail: fmt.Sprintf("is shadowed by an earlier candidate that matches any host at a %d byte floor, so it can never be reached", + unconditionalFloor), + }) + case c.Capability != "": + if prior, seen := previous[c.Capability]; seen && floor > prior { + violations = append(violations, candidateViolation{ + Entry: e.Name, + Candidate: c.Model, + Detail: "raises the VRAM floor after a lower one in the same capability group, so it can never be reached", + }) + } + } + + if c.Capability == "" && (!haveUnconditional || floor < unconditionalFloor) { + unconditionalFloor = floor + haveUnconditional = true + } + previous[c.Capability] = floor + } + } + return violations +} + +// loadGalleryIndex parses gallery/index.yaml once for the whole suite. The +// index carries well over a thousand entries, so re-parsing it per spec is +// pure overhead. +var loadGalleryIndex = sync.OnceValues(func() ([]gallery.GalleryModel, error) { + data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) + if err != nil { + return nil, err + } + var entries []gallery.GalleryModel + if err := yaml.Unmarshal(data, &entries); err != nil { + return nil, err + } + return entries, nil +}) + +func plainEntry(name, url string) gallery.GalleryModel { + e := gallery.GalleryModel{} + e.Name = name + e.URL = url + return e +} + +func entryWithCandidates(name, url, minVRAM string, candidates ...gallery.Candidate) gallery.GalleryModel { + e := gallery.GalleryModel{Candidates: candidates, MinVRAM: minVRAM} + e.Name = name + e.URL = url + return e +} + +// candidateFixture builds an entry with candidates plus the entries it +// references. Synthetic fixtures keep the invariant logic covered however few +// entries in gallery/index.yaml declare candidates; without them every +// index-driven spec below is a no-op that passes while checking nothing. +func candidateFixture(base gallery.GalleryModel, referenced ...gallery.GalleryModel) []gallery.GalleryModel { + return append([]gallery.GalleryModel{base}, referenced...) +} + +var _ = Describe("gallery candidate lint helpers", func() { + // An entry declares candidates solely by carrying the key. GalleryBackend.IsMeta() + // has deliberately different semantics (it requires an EMPTY uri), so a + // well-meaning alignment of the two would make every helper skip every + // entry and pass silently. Assert the distinction directly. + It("treats an entry with candidates as such even though it has a url", func() { + Expect(entryWithCandidates("base", "u://base", "2GiB", gallery.Candidate{Model: "big"}).HasCandidates()).To(BeTrue()) + Expect(plainEntry("big", "u://big").HasCandidates()).To(BeFalse()) + }) + + It("passes every invariant on a valid entry", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "4GiB", + gallery.Candidate{Model: "big", Capability: "nvidia", MinVRAM: "24GiB"}, + gallery.Candidate{Model: "mid", Capability: "nvidia", MinVRAM: "12GiB"}, + gallery.Candidate{Model: "metal-big", Capability: "metal", MinVRAM: "32GiB"}, + gallery.Candidate{Model: "small", MinVRAM: "8GiB"}, + ), + plainEntry("big", "u://big"), + plainEntry("mid", "u://mid"), + plainEntry("metal-big", "u://metal-big"), + plainEntry("small", "u://small"), + ) + + Expect(checkCandidateReferences(entries)).To(BeEmpty()) + Expect(checkCandidateFloors(entries)).To(BeEmpty()) + Expect(checkBaseFloor(entries)).To(BeEmpty()) + Expect(checkCandidateCapabilities(entries)).To(BeEmpty()) + Expect(checkCandidateOrdering(entries)).To(BeEmpty()) + }) + + It("passes every invariant on an entry that declares no floor of its own", func() { + // A floorless entry is the weakest possible base, below every + // candidate, which is exactly the position the base should hold. + entries := candidateFixture( + entryWithCandidates("base", "u://base", "", + gallery.Candidate{Model: "big", MinVRAM: "24GiB"}, + ), + plainEntry("big", "u://big"), + ) + + Expect(checkCandidateFloors(entries)).To(BeEmpty()) + Expect(checkBaseFloor(entries)).To(BeEmpty()) + Expect(checkCandidateOrdering(entries)).To(BeEmpty()) + }) + + Describe("checkBaseFloor", func() { + It("flags a candidate whose floor sits below the entry's own", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "20GiB", + gallery.Candidate{Model: "big", MinVRAM: "8GiB"}, + ), + plainEntry("big", "u://big"), + ) + + violations := checkBaseFloor(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("big")) + Expect(violations[0].Detail).To(ContainSubstring("at or below the entry's own")) + }) + + It("flags a candidate whose floor merely equals the entry's own", func() { + // Equal floors make the candidate unreachable just as surely: every + // host clearing it clears the base, and the base is checked first + // only in the sense that first-match never reaches this candidate. + entries := candidateFixture( + entryWithCandidates("base", "u://base", "8GiB", + gallery.Candidate{Model: "big", MinVRAM: "8GiB"}, + ), + plainEntry("big", "u://big"), + ) + + violations := checkBaseFloor(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("big")) + }) + + It("flags an entry whose own min_vram cannot be parsed", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "eight gigs", + gallery.Candidate{Model: "big", MinVRAM: "24GiB"}, + ), + plainEntry("big", "u://big"), + ) + + violations := checkBaseFloor(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Entry).To(Equal("base")) + Expect(violations[0].Candidate).To(BeEmpty()) + Expect(violations[0].Detail).To(ContainSubstring("bad min_vram")) + }) + }) + + Describe("checkCandidateOrdering", func() { + It("flags a candidate shadowed by an earlier unconditional candidate", func() { + // "a" matches any host at 8GiB, so the nvidia candidate at 20GiB is + // dead: every nvidia host clearing 20GiB already cleared 8GiB. + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "a", MinVRAM: "8GiB"}, + gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"}, + ), + plainEntry("a", "u://a"), plainEntry("b", "u://b"), + ) + + violations := checkCandidateOrdering(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("b")) + Expect(violations[0].Detail).To(ContainSubstring("shadowed by an earlier candidate that matches any host")) + }) + + It("flags a floor inversion inside one capability group", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"}, + gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"}, + ), + plainEntry("a", "u://a"), plainEntry("b", "u://b"), + ) + + violations := checkCandidateOrdering(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("b")) + Expect(violations[0].Detail).To(ContainSubstring("same capability group")) + }) + + It("keeps distinct capability groups independent", func() { + // A high metal floor after a low nvidia floor is fine: no host + // reports both capabilities. + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"}, + gallery.Candidate{Model: "b", Capability: "metal", MinVRAM: "32GiB"}, + ), + plainEntry("a", "u://a"), plainEntry("b", "u://b"), + ) + + Expect(checkCandidateOrdering(entries)).To(BeEmpty()) + }) + }) + + Describe("checkCandidateFloors", func() { + It("flags a candidate with no floor", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "a", Capability: "nvidia"}, + ), + plainEntry("a", "u://a"), + ) + + violations := checkCandidateFloors(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("a")) + Expect(violations[0].Detail).To(ContainSubstring("needs a min_vram")) + }) + + It("flags a candidate whose floor cannot be parsed", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "a", MinVRAM: "lots"}, + ), + plainEntry("a", "u://a"), + ) + + violations := checkCandidateFloors(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("a")) + Expect(violations[0].Detail).To(ContainSubstring("bad min_vram")) + }) + }) + + Describe("checkCandidateReferences", func() { + It("flags a candidate naming an entry that does not exist", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "ghost", MinVRAM: "20GiB"}, + ), + plainEntry("a", "u://a"), + ) + + violations := checkCandidateReferences(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("ghost")) + Expect(violations[0].Detail).To(ContainSubstring("unknown model")) + }) + + It("flags a candidate naming an entry that declares candidates itself", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "nested", MinVRAM: "20GiB"}, + ), + entryWithCandidates("nested", "u://nested", "4GiB", gallery.Candidate{Model: "a", MinVRAM: "30GiB"}), + plainEntry("a", "u://a"), + ) + + violations := checkCandidateReferences(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("nested")) + Expect(violations[0].Detail).To(ContainSubstring("nesting is not allowed")) + }) + }) + + Describe("checkCandidateCapabilities", func() { + It("flags a capability the system can never report", func() { + entries := candidateFixture( + entryWithCandidates("base", "u://base", "2GiB", + gallery.Candidate{Model: "a", Capability: "cuda", MinVRAM: "20GiB"}, + ), + plainEntry("a", "u://a"), + ) + + violations := checkCandidateCapabilities(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Candidate).To(Equal("a")) + Expect(violations[0].Detail).To(ContainSubstring(`unknown capability "cuda"`)) + }) + + It("flags an unknown capability on the entry itself", func() { + entry := entryWithCandidates("base", "u://base", "2GiB", gallery.Candidate{Model: "a", MinVRAM: "20GiB"}) + entry.Capability = "NVIDIA" + entries := candidateFixture(entry, plainEntry("a", "u://a")) + + violations := checkCandidateCapabilities(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Entry).To(Equal("base")) + Expect(violations[0].Candidate).To(BeEmpty()) + Expect(violations[0].Detail).To(ContainSubstring(`unknown capability "NVIDIA"`)) + }) + }) +}) + +var _ = Describe("gallery/index.yaml candidate invariants", Ordered, func() { + var entries []gallery.GalleryModel + + BeforeAll(func() { + var err error + entries, err = loadGalleryIndex() + Expect(err).ToNot(HaveOccurred()) + // A truncated or emptied index unmarshals cleanly and would make every + // spec below vacuously pass. + Expect(entries).ToNot(BeEmpty()) + }) + + It("references only existing entries that declare no candidates themselves", func() { + v := checkCandidateReferences(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("gives every candidate a VRAM floor", func() { + v := checkCandidateFloors(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("keeps every entry's own floor below its candidates'", func() { + v := checkBaseFloor(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("uses only capabilities the system can report", func() { + v := checkCandidateCapabilities(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("orders candidates so no candidate is shadowed by an earlier one", func() { + v := checkCandidateOrdering(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) +}) diff --git a/core/gallery/meta_install_test.go b/core/gallery/meta_install_test.go deleted file mode 100644 index ee57162a7445..000000000000 --- a/core/gallery/meta_install_test.go +++ /dev/null @@ -1,407 +0,0 @@ -package gallery_test - -import ( - "context" - "os" - "path/filepath" - - "dario.cat/mergo" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "gopkg.in/yaml.v3" - - "github.com/mudler/LocalAI/core/config" - "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/pkg/system" -) - -var _ = Describe("GalleryModel meta entries", func() { - It("is not meta when it has no candidates", func() { - m := gallery.GalleryModel{} - m.Name = "plain" - Expect(m.IsMeta()).To(BeFalse()) - }) - - It("is meta when it has candidates", func() { - m := gallery.GalleryModel{Candidates: []gallery.Candidate{{Model: "x"}}} - Expect(m.IsMeta()).To(BeTrue()) - }) - - It("parses a candidate list from gallery YAML in order", func() { - var m gallery.GalleryModel - err := yaml.Unmarshal([]byte(` -name: qwen3-8b -url: "github:example/repo/qwen3-8b-gguf-q4.yaml@master" -candidates: - - model: qwen3-8b-vllm-awq - capability: nvidia - min_vram: 20GiB - - model: qwen3-8b-gguf-q4 -`), &m) - Expect(err).ToNot(HaveOccurred()) - Expect(m.IsMeta()).To(BeTrue()) - Expect(m.Candidates).To(HaveLen(2)) - Expect(m.Candidates[0].Model).To(Equal("qwen3-8b-vllm-awq")) - Expect(m.Candidates[0].Capability).To(Equal("nvidia")) - Expect(m.Candidates[0].MinVRAM).To(Equal("20GiB")) - Expect(m.Candidates[1].Model).To(Equal("qwen3-8b-gguf-q4")) - Expect(m.Candidates[1].Capability).To(BeEmpty()) - }) -}) - -var _ = Describe("ResolveMetaModel", func() { - gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 } - - newModel := func(name, url, description, icon string) *gallery.GalleryModel { - m := &gallery.GalleryModel{} - m.Name = name - m.URL = url - m.Description = description - m.Icon = icon - return m - } - - var models []*gallery.GalleryModel - var meta *gallery.GalleryModel - - BeforeEach(func() { - concreteVLLM := newModel("qwen3-8b-vllm-awq", "file://vllm.yaml", "AWQ variant", "vllm.png") - concreteGGUF := newModel("qwen3-8b-gguf-q4", "file://gguf.yaml", "GGUF variant", "gguf.png") - meta = newModel("qwen3-8b", "file://gguf.yaml", "Qwen3 8B", "qwen.png") - meta.Tags = []string{"llm"} - meta.Candidates = []gallery.Candidate{ - {Model: "qwen3-8b-vllm-awq", Capability: "nvidia", MinVRAM: "20GiB"}, - {Model: "qwen3-8b-gguf-q4"}, - } - models = []*gallery.GalleryModel{concreteVLLM, concreteGGUF, meta} - }) - - It("installs the concrete payload under the meta's name", func() { - resolved, candidate, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") - Expect(err).ToNot(HaveOccurred()) - Expect(candidate.Model).To(Equal("qwen3-8b-vllm-awq")) - Expect(resolved.Name).To(Equal("qwen3-8b")) - Expect(resolved.URL).To(Equal("file://vllm.yaml")) - }) - - It("presents the meta's metadata, not the variant's", func() { - resolved, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") - Expect(err).ToNot(HaveOccurred()) - Expect(resolved.Description).To(Equal("Qwen3 8B")) - Expect(resolved.Icon).To(Equal("qwen.png")) - Expect(resolved.Tags).To(ConsistOf("llm")) - }) - - It("keeps the name stable when a variant is pinned", func() { - resolved, candidate, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "qwen3-8b-gguf-q4") - Expect(err).ToNot(HaveOccurred()) - Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4")) - Expect(resolved.Name).To(Equal("qwen3-8b")) - Expect(resolved.URL).To(Equal("file://gguf.yaml")) - }) - - It("detaches the resolved entry from the gallery's own maps and slices", func() { - models[0].Overrides = map[string]any{"f16": true} - models[0].AdditionalFiles = []gallery.File{{Filename: "a.bin"}} - - resolved, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") - Expect(err).ToNot(HaveOccurred()) - - // The install path merges the caller's request overrides into this map in - // place, so aliasing the gallery's map would write one caller's request - // into the shared catalog and leak it into every later install. - resolved.Overrides["f16"] = false - resolved.Overrides["threads"] = 4 - Expect(models[0].Overrides).To(HaveKeyWithValue("f16", true)) - Expect(models[0].Overrides).ToNot(HaveKey("threads")) - - resolved.AdditionalFiles[0].Filename = "mutated.bin" - Expect(models[0].AdditionalFiles[0].Filename).To(Equal("a.bin")) - - resolved.Tags = append(resolved.Tags, "extra") - Expect(meta.Tags).To(ConsistOf("llm")) - }) - - It("detaches nested override maps from the gallery's own entry", func() { - models[0].Overrides = map[string]any{ - "parameters": map[string]any{"model": "real-variant.gguf"}, - "stopwords": []any{""}, - } - - resolved, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") - Expect(err).ToNot(HaveOccurred()) - - // Cloning only the top level would leave this inner map shared with the - // gallery entry, so writing through the resolved copy would rewrite the - // catalog's own payload. - resolved.Overrides["parameters"].(map[string]any)["model"] = "callers-choice.gguf" - resolved.Overrides["stopwords"].([]any)[0] = "<|im_end|>" - - Expect(models[0].Overrides["parameters"]).To(HaveKeyWithValue("model", "real-variant.gguf")) - Expect(models[0].Overrides["stopwords"]).To(Equal([]any{""})) - }) - - It("does not write the caller's overrides back into the gallery entry", func() { - models[0].Overrides = map[string]any{"parameters": map[string]any{"model": "real-variant.gguf"}} - - resolved, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") - Expect(err).ToNot(HaveOccurred()) - - // This is exactly what the install path does with the caller's request - // overrides, and mergo recurses into nested maps and overwrites them in - // place. Asserting against the in-memory catalog is the only way to - // observe the leak: re-reading the gallery from disk re-unmarshals fresh - // maps and would pass whether or not the resolved entry was detached. - requestOverrides := map[string]any{"parameters": map[string]any{"model": "callers-choice.gguf"}} - Expect(mergo.Merge(&resolved.Overrides, requestOverrides, mergo.WithOverride)).To(Succeed()) - - Expect(resolved.Overrides["parameters"]).To(HaveKeyWithValue("model", "callers-choice.gguf")) - Expect(models[0].Overrides["parameters"]).To(HaveKeyWithValue("model", "real-variant.gguf")) - }) - - It("errors when a candidate references a missing entry", func() { - meta.Candidates = []gallery.Candidate{{Model: "does-not-exist"}} - _, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("does-not-exist")) - }) - - It("refuses a candidate that is itself a meta entry", func() { - nested := newModel("nested", "", "", "") - nested.Candidates = []gallery.Candidate{{Model: "qwen3-8b-gguf-q4"}} - models = append(models, nested) - meta.Candidates = []gallery.Candidate{{Model: "nested"}} - _, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("nested")) - }) - - It("surfaces a bad pin", func() { - _, _, err := gallery.ResolveMetaModel(models, meta, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "nope") - Expect(err).To(MatchError(gallery.ErrPinNotFound)) - }) -}) - -var _ = Describe("InstallModelFromGallery with meta entries", func() { - var tempdir string - var galleries []config.Gallery - var systemState *system.SystemState - - // The variants are described with an inline config_file rather than a URL so - // the whole install runs off the local filesystem with no network access. - // The meta entry keeps a url as well, because a real meta entry carries one - // as a fallback for older LocalAI releases that do not understand candidates, - // and installing that fallback instead of a variant is exactly the regression - // these specs guard against. - newGallery := func(meta gallery.GalleryModel, variants ...gallery.GalleryModel) { - fallback := gallery.ModelConfig{ - Name: "legacy-fallback", - Description: "legacy fallback payload", - ConfigFile: "backend: fallback-backend\n", - } - fallbackYAML, err := yaml.Marshal(fallback) - Expect(err).ToNot(HaveOccurred()) - fallbackPath := filepath.Join(tempdir, "fallback.yaml") - Expect(os.WriteFile(fallbackPath, fallbackYAML, 0600)).To(Succeed()) - - meta.URL = "file://" + fallbackPath - entries := append([]gallery.GalleryModel{meta}, variants...) - - out, err := yaml.Marshal(entries) - Expect(err).ToNot(HaveOccurred()) - galleryPath := filepath.Join(tempdir, "gallery.yaml") - Expect(os.WriteFile(galleryPath, out, 0600)).To(Succeed()) - - galleries = []config.Gallery{{Name: "test", URL: "file://" + galleryPath}} - } - - variant := func(name, backend string) gallery.GalleryModel { - m := gallery.GalleryModel{ConfigFile: map[string]any{"backend": backend}} - m.Name = name - m.Description = "variant " + name - return m - } - - // urlVariant describes a variant through a url rather than an inline - // config_file. The distinction matters for the recorded name: the url branch - // reads a name out of the fetched config, and that name is the variant's own, - // so only the meta-name overlay can keep the record under the meta's name. - // The inline config_file branch seeds the name from the already-renamed - // resolved entry and so cannot observe the overlay at all. - urlVariant := func(name, backend string) gallery.GalleryModel { - payload := gallery.ModelConfig{ - Name: name, - Description: "variant " + name, - ConfigFile: "backend: " + backend + "\n", - } - out, err := yaml.Marshal(payload) - Expect(err).ToNot(HaveOccurred()) - payloadPath := filepath.Join(tempdir, "payload-"+name+".yaml") - Expect(os.WriteFile(payloadPath, out, 0600)).To(Succeed()) - - m := gallery.GalleryModel{} - m.Name = name - m.Description = "variant " + name - m.URL = "file://" + payloadPath - return m - } - - metaEntry := func(name string, candidates ...string) gallery.GalleryModel { - m := gallery.GalleryModel{} - m.Name = name - m.Description = "the meta entry" - m.Icon = "meta.png" - for _, c := range candidates { - m.Candidates = append(m.Candidates, gallery.Candidate{Model: c}) - } - return m - } - - install := func(name string, req gallery.GalleryModel, options ...gallery.InstallOption) error { - return gallery.InstallModelFromGallery( - context.TODO(), galleries, []config.Gallery{}, systemState, nil, - name, req, func(string, string, string, float64) {}, false, false, false, options...) - } - - installedBackend := func(name string) string { - dat, err := os.ReadFile(filepath.Join(tempdir, name+".yaml")) - Expect(err).ToNot(HaveOccurred()) - content := map[string]any{} - Expect(yaml.Unmarshal(dat, &content)).To(Succeed()) - return content["backend"].(string) - } - - BeforeEach(func() { - var err error - tempdir, err = os.MkdirTemp("", "meta-install") - Expect(err).ToNot(HaveOccurred()) - DeferCleanup(func() { Expect(os.RemoveAll(tempdir)).To(Succeed()) }) - - systemState, err = system.GetSystemState(system.WithModelPath(tempdir)) - Expect(err).ToNot(HaveOccurred()) - - newGallery( - metaEntry("qwen3-8b", "qwen3-8b-variant-a", "qwen3-8b-variant-b"), - variant("qwen3-8b-variant-a", "variant-a-backend"), - variant("qwen3-8b-variant-b", "variant-b-backend"), - ) - }) - - It("installs the resolved variant's payload, not the meta's url fallback", func() { - Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) - - // If meta-ness stopped winning over the url, this would be - // "fallback-backend" and every meta entry would silently install the - // legacy payload instead of a hardware-appropriate variant. - Expect(installedBackend("qwen3-8b")).To(Equal("variant-a-backend")) - }) - - It("round-trips the resolution record to disk under the meta's name", func() { - // The variant is described by url so its payload carries its own name. - // Without the meta-name overlay the record persists as - // "qwen3-8b-variant-a", and the stable name a meta entry exists to - // provide is lost the moment anything reads the record back. - newGallery( - metaEntry("qwen3-8b", "qwen3-8b-variant-a", "qwen3-8b-variant-b"), - urlVariant("qwen3-8b-variant-a", "variant-a-backend"), - urlVariant("qwen3-8b-variant-b", "variant-b-backend"), - ) - - Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) - - record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") - Expect(err).ToNot(HaveOccurred()) - Expect(record.MetaName).To(Equal("qwen3-8b")) - Expect(record.ResolvedVariant).To(Equal("qwen3-8b-variant-a")) - Expect(record.PinnedVariant).To(BeEmpty()) - Expect(record.Name).To(Equal("qwen3-8b")) - Expect(record.Description).To(Equal("the meta entry")) - Expect(installedBackend("qwen3-8b")).To(Equal("variant-a-backend")) - }) - - It("records a pin and honors it on a plain reinstall", func() { - Expect(install("qwen3-8b", gallery.GalleryModel{}, gallery.WithVariant("qwen3-8b-variant-b"))).To(Succeed()) - - record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") - Expect(err).ToNot(HaveOccurred()) - Expect(record.PinnedVariant).To(Equal("qwen3-8b-variant-b")) - Expect(record.ResolvedVariant).To(Equal("qwen3-8b-variant-b")) - Expect(installedBackend("qwen3-8b")).To(Equal("variant-b-backend")) - - // No WithVariant this time: hardware resolution would pick variant-a, so - // only the recalled pin can keep this on variant-b. - Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) - Expect(installedBackend("qwen3-8b")).To(Equal("variant-b-backend")) - - record, err = gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") - Expect(err).ToNot(HaveOccurred()) - Expect(record.PinnedVariant).To(Equal("qwen3-8b-variant-b")) - }) - - It("honors a pin recorded under a custom install name", func() { - req := gallery.GalleryModel{} - req.Name = "prod-llm" - - Expect(install("qwen3-8b", req, gallery.WithVariant("qwen3-8b-variant-b"))).To(Succeed()) - - // The pin is written under the installed name, so the recall must read it - // back under that name too and not under the gallery entry's name. - record, err := gallery.GetLocalModelConfiguration(tempdir, "prod-llm") - Expect(err).ToNot(HaveOccurred()) - Expect(record.PinnedVariant).To(Equal("qwen3-8b-variant-b")) - Expect(record.MetaName).To(Equal("qwen3-8b")) - Expect(installedBackend("prod-llm")).To(Equal("variant-b-backend")) - - Expect(install("qwen3-8b", req)).To(Succeed()) - Expect(installedBackend("prod-llm")).To(Equal("variant-b-backend")) - }) - - It("writes each declared url only once into the persisted gallery file", func() { - meta := metaEntry("qwen3-8b", "qwen3-8b-variant-a") - meta.URLs = []string{"https://example.invalid/qwen3"} - newGallery(meta, variant("qwen3-8b-variant-a", "variant-a-backend")) - - Expect(install("qwen3-8b", gallery.GalleryModel{})).To(Succeed()) - - record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b") - Expect(err).ToNot(HaveOccurred()) - Expect(record.URLs).To(ConsistOf("https://example.invalid/qwen3")) - }) -}) - -var _ = Describe("legacy client compatibility", func() { - It("exposes a url on every meta entry to clients that ignore candidates", func() { - data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) - Expect(err).ToNot(HaveOccurred()) - - // Parse exactly as an older LocalAI release would: non-strictly, with - // no knowledge of the candidates key. - var legacy []struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - } - Expect(yaml.Unmarshal(data, &legacy)).To(Succeed()) - - var current []gallery.GalleryModel - Expect(yaml.Unmarshal(data, ¤t)).To(Succeed()) - - urlByName := map[string]string{} - for _, e := range legacy { - urlByName[e.Name] = e.URL - } - - metaCount := 0 - for _, e := range current { - if !e.IsMeta() { - continue - } - metaCount++ - // Without a url an old client lists the entry and installs an - // empty model, because it silently drops candidates. - Expect(urlByName[e.Name]).ToNot(BeEmpty(), - "meta entry %q is invisible payload-wise to older clients", e.Name) - } - Expect(metaCount).To(BeNumerically(">", 0), - "expected at least one meta entry in the gallery index") - }) -}) diff --git a/core/gallery/meta_lint_test.go b/core/gallery/meta_lint_test.go deleted file mode 100644 index d964f14570ba..000000000000 --- a/core/gallery/meta_lint_test.go +++ /dev/null @@ -1,510 +0,0 @@ -package gallery_test - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "sync" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "gopkg.in/yaml.v3" - - "github.com/mudler/LocalAI/core/gallery" -) - -// knownCapabilities mirrors the values SystemState.DetectedCapability() can -// actually report, which is what the candidate resolver compares against with -// a case-sensitive exact match. A capability outside this set can never match, -// so a typo would silently make a candidate unreachable on every host. -// -// Note "cpu" is deliberately absent: it exists only as a fallback key inside -// SystemState.Capability(capMap) for meta backends, and is never a value -// getSystemCapabilities() returns. A CPU-only host reports "default". -var knownCapabilities = map[string]bool{ - "default": true, "metal": true, "darwin-x86": true, - "nvidia": true, "nvidia-cuda-12": true, "nvidia-cuda-13": true, - "nvidia-l4t": true, "nvidia-l4t-cuda-12": true, "nvidia-l4t-cuda-13": true, - "intel": true, "amd": true, "vulkan": true, -} - -// metaViolation is one invariant breach found by a lint helper. Helpers return -// every breach they find instead of stopping at the first, so a single run -// names them all rather than forcing a fix-one-rerun-repeat cycle. -type metaViolation struct { - Entry string - Candidate string - Detail string -} - -func (v metaViolation) String() string { - if v.Candidate == "" { - return fmt.Sprintf("%s: %s", v.Entry, v.Detail) - } - return fmt.Sprintf("%s -> candidate %q: %s", v.Entry, v.Candidate, v.Detail) -} - -func formatViolations(violations []metaViolation) string { - lines := make([]string, 0, len(violations)) - for _, v := range violations { - lines = append(lines, " "+v.String()) - } - return "\n" + strings.Join(lines, "\n") -} - -func indexEntriesByName(entries []gallery.GalleryModel) map[string]gallery.GalleryModel { - byName := make(map[string]gallery.GalleryModel, len(entries)) - for _, e := range entries { - byName[e.Name] = e - } - return byName -} - -// checkMetaFallbackURL verifies that released LocalAI versions, which ignore -// the candidates key entirely, still install something sensible: the meta -// entry needs a url, and it must be the final candidate's url so old and new -// clients agree on the least demanding option. -func checkMetaFallbackURL(entries []gallery.GalleryModel) []metaViolation { - byName := indexEntriesByName(entries) - var violations []metaViolation - for _, e := range entries { - if !e.IsMeta() { - continue - } - if e.URL == "" { - violations = append(violations, metaViolation{Entry: e.Name, Detail: "needs a url fallback for older clients"}) - } - if len(e.ConfigFile) > 0 { - violations = append(violations, metaViolation{Entry: e.Name, Detail: "must not carry an inline config_file"}) - } - if len(e.AdditionalFiles) > 0 { - violations = append(violations, metaViolation{Entry: e.Name, Detail: "must not carry files"}) - } - - last := e.Candidates[len(e.Candidates)-1] - fallback, ok := byName[last.Model] - if !ok { - // checkMetaReferences owns reporting the dangling name; there is - // nothing to compare the url against here. - continue - } - if e.URL != fallback.URL { - violations = append(violations, metaViolation{ - Entry: e.Name, - Candidate: last.Model, - Detail: "meta url must equal its final candidate url so old and new clients agree", - }) - } - } - return violations -} - -// checkMetaReferences verifies every candidate names an existing, concrete -// entry. Resolution is a single pass, so a nested meta reference would install -// an entry that carries no files. -func checkMetaReferences(entries []gallery.GalleryModel) []metaViolation { - byName := indexEntriesByName(entries) - var violations []metaViolation - for _, e := range entries { - if !e.IsMeta() { - continue - } - for _, c := range e.Candidates { - target, ok := byName[c.Model] - if !ok { - violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "references unknown model"}) - continue - } - if target.IsMeta() { - violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "references a meta entry; nesting is not allowed"}) - } - } - } - return violations -} - -// checkMetaConstraints verifies that only the final candidate is an -// unconstrained last resort. An earlier candidate without a floor would -// capture every host and make everything after it dead. -func checkMetaConstraints(entries []gallery.GalleryModel) []metaViolation { - var violations []metaViolation - for _, e := range entries { - if !e.IsMeta() { - continue - } - for i, c := range e.Candidates { - _, declared, err := c.EffectiveMinVRAM() - if err != nil { - violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "has a bad min_vram: " + err.Error()}) - continue - } - - if i == len(e.Candidates)-1 { - if declared { - violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "final candidate must be an unconstrained last resort"}) - } - if c.Capability != "" { - violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "final candidate must not require a capability"}) - } - continue - } - if !declared { - violations = append(violations, metaViolation{Entry: e.Name, Candidate: c.Model, Detail: "needs a min_vram; the nightly job should have inferred one"}) - } - } - } - return violations -} - -// checkMetaCapabilities verifies candidates only name capabilities the system -// can actually report, since the resolver compares them exactly. -func checkMetaCapabilities(entries []gallery.GalleryModel) []metaViolation { - var violations []metaViolation - for _, e := range entries { - if !e.IsMeta() { - continue - } - for _, c := range e.Candidates { - if c.Capability == "" { - continue - } - if !knownCapabilities[c.Capability] { - violations = append(violations, metaViolation{ - Entry: e.Name, - Candidate: c.Model, - Detail: fmt.Sprintf("uses unknown capability %q", c.Capability), - }) - } - } - } - return violations -} - -// checkMetaOrdering verifies no candidate is shadowed by an earlier one. -// Selection is first-match over the authored order, so a candidate is dead -// whenever an earlier one matches every host this one would. -// -// Two shapes cause that. Within a single capability group the groups are -// mutually exclusive, so only a floor rising above an earlier floor is -// unreachable. A candidate with an EMPTY capability matches every host and so -// dominates ACROSS groups: every later candidate whose floor is at or above -// the running minimum unconditional floor is dead, whatever capability it asks -// for. Tracking that running minimum subsumes the same-group check for the -// empty capability. -func checkMetaOrdering(entries []gallery.GalleryModel) []metaViolation { - var violations []metaViolation - for _, e := range entries { - if !e.IsMeta() { - continue - } - previous := map[string]uint64{} - var unconditionalFloor uint64 - haveUnconditional := false - - for _, c := range e.Candidates { - floor, declared, err := c.EffectiveMinVRAM() - if err != nil || !declared { - // Parse errors and absent floors belong to checkMetaConstraints. - continue - } - - switch { - case haveUnconditional && floor >= unconditionalFloor: - violations = append(violations, metaViolation{ - Entry: e.Name, - Candidate: c.Model, - Detail: fmt.Sprintf("is shadowed by an earlier candidate that matches any host at a %d byte floor, so it can never be reached", - unconditionalFloor), - }) - case c.Capability != "": - if prior, seen := previous[c.Capability]; seen && floor > prior { - violations = append(violations, metaViolation{ - Entry: e.Name, - Candidate: c.Model, - Detail: "raises the VRAM floor after a lower one in the same capability group, so it can never be reached", - }) - } - } - - if c.Capability == "" && (!haveUnconditional || floor < unconditionalFloor) { - unconditionalFloor = floor - haveUnconditional = true - } - previous[c.Capability] = floor - } - } - return violations -} - -// loadGalleryIndex parses gallery/index.yaml once for the whole suite. The -// index carries well over a thousand entries, so re-parsing it per spec is -// pure overhead. -var loadGalleryIndex = sync.OnceValues(func() ([]gallery.GalleryModel, error) { - data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) - if err != nil { - return nil, err - } - var entries []gallery.GalleryModel - if err := yaml.Unmarshal(data, &entries); err != nil { - return nil, err - } - return entries, nil -}) - -func concreteEntry(name, url string) gallery.GalleryModel { - e := gallery.GalleryModel{} - e.Name = name - e.URL = url - return e -} - -func metaEntry(name, url string, candidates ...gallery.Candidate) gallery.GalleryModel { - e := gallery.GalleryModel{Candidates: candidates} - e.Name = name - e.URL = url - return e -} - -// metaFixture builds a meta entry plus the concrete entries it references. -// Synthetic fixtures keep the invariant logic covered while gallery/index.yaml -// still holds zero meta entries; without them every index-driven spec below is -// a no-op that passes while checking nothing. -func metaFixture(meta gallery.GalleryModel, concrete ...gallery.GalleryModel) []gallery.GalleryModel { - return append([]gallery.GalleryModel{meta}, concrete...) -} - -var _ = Describe("meta entry lint helpers", func() { - // A model entry is meta solely by carrying candidates. GalleryBackend.IsMeta() - // has deliberately opposite semantics (it requires an EMPTY uri), so a - // well-meaning alignment of the two would make every helper skip every - // entry and pass silently. Assert the distinction directly. - It("treats an entry with candidates as meta even though it has a url", func() { - Expect(metaEntry("meta", "u://big", gallery.Candidate{Model: "big"}).IsMeta()).To(BeTrue()) - Expect(concreteEntry("big", "u://big").IsMeta()).To(BeFalse()) - }) - - It("passes every invariant on a valid meta entry", func() { - entries := metaFixture( - metaEntry("meta", "u://small", - gallery.Candidate{Model: "big", Capability: "nvidia", MinVRAM: "24GiB"}, - gallery.Candidate{Model: "mid", Capability: "nvidia", MinVRAM: "12GiB"}, - gallery.Candidate{Model: "metal-big", Capability: "metal", MinVRAM: "32GiB"}, - gallery.Candidate{Model: "small"}, - ), - concreteEntry("big", "u://big"), - concreteEntry("mid", "u://mid"), - concreteEntry("metal-big", "u://metal-big"), - concreteEntry("small", "u://small"), - ) - - Expect(checkMetaFallbackURL(entries)).To(BeEmpty()) - Expect(checkMetaReferences(entries)).To(BeEmpty()) - Expect(checkMetaConstraints(entries)).To(BeEmpty()) - Expect(checkMetaCapabilities(entries)).To(BeEmpty()) - Expect(checkMetaOrdering(entries)).To(BeEmpty()) - }) - - Describe("checkMetaOrdering", func() { - It("flags a candidate shadowed by an earlier unconditional candidate", func() { - // "a" matches any host at 8GiB, so the nvidia candidate at 20GiB is - // dead: every nvidia host clearing 20GiB already cleared 8GiB. - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", MinVRAM: "8GiB"}, - gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("a", "u://a"), concreteEntry("b", "u://b"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaOrdering(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("b")) - Expect(violations[0].Detail).To(ContainSubstring("shadowed by an earlier candidate that matches any host")) - }) - - It("flags a floor inversion inside one capability group", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"}, - gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("a", "u://a"), concreteEntry("b", "u://b"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaOrdering(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("b")) - Expect(violations[0].Detail).To(ContainSubstring("same capability group")) - }) - - It("keeps distinct capability groups independent", func() { - // A high metal floor after a low nvidia floor is fine: no host - // reports both capabilities. - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"}, - gallery.Candidate{Model: "b", Capability: "metal", MinVRAM: "32GiB"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("a", "u://a"), concreteEntry("b", "u://b"), concreteEntry("c", "u://c"), - ) - - Expect(checkMetaOrdering(entries)).To(BeEmpty()) - }) - }) - - Describe("checkMetaConstraints", func() { - It("flags a non-final candidate with no floor", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", Capability: "nvidia"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaConstraints(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("a")) - Expect(violations[0].Detail).To(ContainSubstring("needs a min_vram")) - }) - - It("flags a final candidate that declares a floor", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c", MinVRAM: "8GiB"}, - ), - concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaConstraints(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("c")) - Expect(violations[0].Detail).To(ContainSubstring("unconstrained last resort")) - }) - - It("flags a final candidate that requires a capability", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c", Capability: "nvidia"}, - ), - concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaConstraints(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("c")) - Expect(violations[0].Detail).To(ContainSubstring("must not require a capability")) - }) - }) - - Describe("checkMetaReferences", func() { - It("flags a candidate naming an entry that does not exist", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "ghost", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("c", "u://c"), - ) - - violations := checkMetaReferences(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("ghost")) - Expect(violations[0].Detail).To(ContainSubstring("unknown model")) - }) - - It("flags a candidate naming another meta entry", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "nested", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c"}, - ), - metaEntry("nested", "u://c", gallery.Candidate{Model: "c"}), - concreteEntry("c", "u://c"), - ) - - violations := checkMetaReferences(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("nested")) - Expect(violations[0].Detail).To(ContainSubstring("nesting is not allowed")) - }) - }) - - Describe("checkMetaFallbackURL", func() { - It("flags a meta url that differs from its final candidate url", func() { - entries := metaFixture( - metaEntry("meta", "u://wrong", - gallery.Candidate{Model: "a", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaFallbackURL(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("c")) - Expect(violations[0].Detail).To(ContainSubstring("must equal its final candidate url")) - }) - }) - - Describe("checkMetaCapabilities", func() { - It("flags a capability the system can never report", func() { - entries := metaFixture( - metaEntry("meta", "u://c", - gallery.Candidate{Model: "a", Capability: "cuda", MinVRAM: "20GiB"}, - gallery.Candidate{Model: "c"}, - ), - concreteEntry("a", "u://a"), concreteEntry("c", "u://c"), - ) - - violations := checkMetaCapabilities(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("a")) - Expect(violations[0].Detail).To(ContainSubstring(`unknown capability "cuda"`)) - }) - }) -}) - -var _ = Describe("gallery/index.yaml meta entry invariants", Ordered, func() { - var entries []gallery.GalleryModel - - BeforeAll(func() { - var err error - entries, err = loadGalleryIndex() - Expect(err).ToNot(HaveOccurred()) - // A truncated or emptied index unmarshals cleanly and would make every - // spec below vacuously pass. - Expect(entries).ToNot(BeEmpty()) - }) - - It("gives every meta entry a legacy url and no inline payload", func() { - v := checkMetaFallbackURL(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) - - It("references only existing, non-meta entries", func() { - v := checkMetaReferences(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) - - It("constrains every candidate except an unconstrained last resort", func() { - v := checkMetaConstraints(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) - - It("uses only capabilities the system can report", func() { - v := checkMetaCapabilities(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) - - It("orders candidates so no candidate is shadowed by an earlier one", func() { - v := checkMetaOrdering(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) -}) diff --git a/core/gallery/model_artifacts.go b/core/gallery/model_artifacts.go index 01de02d032ba..d323293e6ba0 100644 --- a/core/gallery/model_artifacts.go +++ b/core/gallery/model_artifacts.go @@ -32,8 +32,10 @@ func WithArtifactMaterializer(materializer ArtifactMaterializer) InstallOption { } } -// WithVariant pins a meta model entry to a specific candidate by name, -// bypassing hardware-based resolution. Ignored for non-meta entries. +// WithVariant pins a gallery entry to a specific candidate by name, bypassing +// hardware-based resolution. The entry's own name is a valid pin, since the +// entry is itself the last-resort candidate. Ignored for entries that declare +// no candidates. func WithVariant(variant string) InstallOption { return func(options *installOptions) { options.variant = variant diff --git a/core/gallery/models.go b/core/gallery/models.go index 252dc539c806..917e3861826f 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -61,10 +61,10 @@ type ModelConfig struct { Files []File `yaml:"files"` PromptTemplates []PromptTemplate `yaml:"prompt_templates"` - // The fields below record how a meta entry was resolved, so a reinstall - // or upgrade can honor the same pin and so operators can see which - // variant a stable model name is actually backed by. - MetaName string `yaml:"meta_name,omitempty"` + // The fields below record how an entry carrying candidates was resolved, + // so a reinstall or upgrade can honor the same pin and so operators can + // see which variant a stable model name is actually backed by. + EntryName string `yaml:"entry_name,omitempty"` ResolvedVariant string `yaml:"resolved_variant,omitempty"` PinnedVariant string `yaml:"pinned_variant,omitempty"` } @@ -80,26 +80,48 @@ type PromptTemplate struct { Content string `yaml:"content"` } -// ResolveMetaModel turns a meta gallery entry into the concrete entry that -// should be installed on this host, returning it renamed to the meta's name -// and carrying the meta's presentation metadata. +// effectiveCandidates returns the candidate list resolution actually runs over: +// the declared upgrades followed by the entry itself. +// +// The entry is appended rather than special-cased so there is exactly one +// selection path. Being last makes it the last resort, which is what keeps the +// entry always installable: the list can never be exhausted without reaching it. +func effectiveCandidates(entry *GalleryModel) []Candidate { + candidates := make([]Candidate, 0, len(entry.Candidates)+1) + candidates = append(candidates, entry.Candidates...) + return append(candidates, Candidate{ + Model: entry.Name, + Capability: entry.Capability, + MinVRAM: entry.MinVRAM, + }) +} + +// ResolveVariant picks the gallery entry to install for a host, from the +// upgrades an entry declares plus the entry itself, and returns it renamed to +// the entry's name and carrying the entry's presentation metadata. // // Why the metadata split: the payload (url, config_file, files, overrides) -// must come from the variant because that is what actually gets downloaded, -// while the presentation (name, description, icon, tags) must come from the -// meta so the installed model presents as the model rather than the variant. -func ResolveMetaModel(models []*GalleryModel, meta *GalleryModel, env ResolveEnv, pin string) (*GalleryModel, Candidate, error) { - candidate, err := ResolveCandidate(meta.Candidates, env, pin) +// must come from the chosen variant because that is what actually gets +// downloaded, while the presentation (name, description, icon, tags) must come +// from the entry the user asked for, so the installed model presents as that +// model rather than as one of its variants. +// +// The base entry always resolves. When even its own predicates fall short this +// warns and installs it regardless, because the entry is a complete entry that +// every older LocalAI release installs unconditionally; refusing it here would +// make the gallery behave worse the newer the client is. +func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, pin string) (*GalleryModel, Candidate, error) { + candidates := effectiveCandidates(entry) + base := candidates[len(candidates)-1] + + candidate, err := ResolveCandidate(candidates, env, pin) if err != nil { - return nil, Candidate{}, fmt.Errorf("resolving variant for model %q: %w", meta.Name, err) - } - - concrete := FindGalleryElement(models, candidate.Model) - if concrete == nil { - return nil, Candidate{}, fmt.Errorf("model %q references variant %q which does not exist in any configured gallery", meta.Name, candidate.Model) - } - if concrete.IsMeta() { - return nil, Candidate{}, fmt.Errorf("model %q references variant %q which is itself a meta entry; meta entries may not nest", meta.Name, candidate.Model) + if !errors.Is(err, ErrNoCandidateMatch) { + return nil, Candidate{}, fmt.Errorf("resolving variant for model %q: %w", entry.Name, err) + } + xlog.Warn("This system does not meet the requirements this model declares for itself; installing it anyway because it is the last resort", + "model", entry.Name, "capability", env.Capability, "available_vram", env.VRAM, "reason", err) + candidate = base } // A pin is an operator override and deliberately bypasses the hardware @@ -108,19 +130,41 @@ func ResolveMetaModel(models []*GalleryModel, meta *GalleryModel, env ResolveEnv // It is warned about only once the pin is known to name a real, installable // entry, otherwise a pin naming a listed-but-nonexistent variant would warn // about VRAM and then fail for an entirely unrelated reason. - if pin != "" { + warnPin := func() { + if pin == "" { + return + } if floor, declared, verr := candidate.EffectiveMinVRAM(); verr == nil && declared && env.VRAM < floor { xlog.Warn("Pinned model variant declares more VRAM than this system reports; installing anyway because the pin overrides hardware resolution", - "model", meta.Name, "variant", candidate.Model, "required_vram", floor, "available_vram", env.VRAM) + "model", entry.Name, "variant", candidate.Model, "required_vram", floor, "available_vram", env.VRAM) + } + } + + // Resolving to the base means installing the entry's own payload, which is + // the entry itself; there is no second entry to look up. + source := entry + if candidate.Model != entry.Name { + source = FindGalleryElement(models, candidate.Model) + if source == nil { + return nil, Candidate{}, fmt.Errorf("model %q references variant %q which does not exist in any configured gallery", entry.Name, candidate.Model) + } + if source.HasCandidates() { + return nil, Candidate{}, fmt.Errorf("model %q references variant %q which declares candidates of its own; resolution is a single pass, so those would be silently ignored", entry.Name, candidate.Model) } } + warnPin() - resolved := *concrete - resolved.Name = meta.Name - resolved.Description = meta.Description - resolved.Icon = meta.Icon - resolved.License = meta.License + resolved := *source + resolved.Name = entry.Name + resolved.Description = entry.Description + resolved.Icon = entry.Icon + resolved.License = entry.License + // The resolved entry is a concrete install target, so it must not carry the + // selection fields any more; leaving them would let a second pass resolve + // the already-resolved entry all over again. resolved.Candidates = nil + resolved.Capability = "" + resolved.MinVRAM = "" // The struct copy above is shallow, so every reference-typed field still // aliases the gallery's own entries. The install path mutates Overrides in @@ -135,11 +179,11 @@ func ResolveMetaModel(models []*GalleryModel, meta *GalleryModel, env ResolveEnv // maps and overwrites them in place, so a top-level clone would still hand // the caller the gallery's own inner maps. The slices below hold value types, // so cloning them once fully detaches them. - resolved.Overrides = deepCopyStringMap(concrete.Overrides) - resolved.ConfigFile = deepCopyStringMap(concrete.ConfigFile) - resolved.AdditionalFiles = slices.Clone(concrete.AdditionalFiles) - resolved.URLs = slices.Clone(meta.URLs) - resolved.Tags = slices.Clone(meta.Tags) + resolved.Overrides = deepCopyStringMap(source.Overrides) + resolved.ConfigFile = deepCopyStringMap(source.ConfigFile) + resolved.AdditionalFiles = slices.Clone(source.AdditionalFiles) + resolved.URLs = slices.Clone(entry.URLs) + resolved.Tags = slices.Clone(entry.Tags) return &resolved, candidate, nil } @@ -228,13 +272,13 @@ func InstallModelFromGallery( } if record != nil { - config.MetaName = record.MetaName + config.EntryName = record.EntryName config.ResolvedVariant = record.ResolvedVariant config.PinnedVariant = record.PinnedVariant // The variant's own name would otherwise be persisted here, which - // contradicts the whole point of a meta entry: the model is known by - // the meta's stable name regardless of which variant backs it. - config.Name = record.MetaName + // contradicts the whole point of candidates: the model is known by + // the entry's stable name regardless of which variant backs it. + config.Name = record.EntryName } installName := model.Name @@ -281,10 +325,10 @@ func InstallModelFromGallery( return fmt.Errorf("no model found with name %q", name) } - // Meta-ness is checked before anything looks at the URL: a meta entry also - // carries a url as a fallback for older LocalAI releases that do not - // understand candidates, so carrying both is normal and meta wins here. - if !model.IsMeta() { + // An entry without candidates is installed directly. An entry with them is + // still installable as-is; resolution below only decides whether one of its + // declared upgrades fits this host better. + if !model.HasCandidates() { return applyModel(model, nil) } @@ -296,7 +340,7 @@ func InstallModelFromGallery( // The record is keyed by the name the model was installed under, not by the // gallery entry name: applyModel writes it to ._gallery_.yaml, // where installName is req.Name whenever the caller supplied one. Reading it - // back under the meta's own name would miss the record for every custom-named + // back under the entry's own name would miss the record for every custom-named // install and silently re-resolve a deliberately pinned model onto a // different variant, possibly swapping its backend. if pin == "" { @@ -314,17 +358,17 @@ func InstallModelFromGallery( VRAM: systemState.VRAM, } - resolved, candidate, err := ResolveMetaModel(models, model, env, pin) + resolved, candidate, err := ResolveVariant(models, model, env, pin) if err != nil { return err } - xlog.Info("Resolved meta model to variant", + xlog.Info("Resolved model to variant", "model", model.Name, "variant", candidate.Model, "capability", env.Capability, "vram", env.VRAM, "pinned", pin != "") return applyModel(resolved, &ModelConfig{ - MetaName: model.Name, + EntryName: model.Name, ResolvedVariant: candidate.Model, PinnedVariant: pin, }) diff --git a/core/gallery/models_types.go b/core/gallery/models_types.go index b50dfbcc144a..3b570d8cc02f 100644 --- a/core/gallery/models_types.go +++ b/core/gallery/models_types.go @@ -15,10 +15,19 @@ type GalleryModel struct { ConfigFile map[string]any `json:"config_file,omitempty" yaml:"config_file,omitempty"` // Overrides are used to override the configuration of the model located at URL Overrides map[string]any `json:"overrides,omitempty" yaml:"overrides,omitempty"` - // Candidates, when non-empty, makes this a meta entry: an ordered list of - // concrete gallery entries, the first of which the host satisfies is what - // gets installed. Mirrors GalleryBackend's capabilities map one level up. + // Candidates is an optional ordered list of hardware-gated UPGRADES over + // this entry. The entry itself is always the last-resort candidate, so an + // entry carrying candidates stays a complete, installable entry and older + // LocalAI releases, which drop this key, install it exactly as before. Candidates []Candidate `json:"candidates,omitempty" yaml:"candidates,omitempty"` + // Capability, when set, is the host capability this entry's own payload + // prefers. It is advisory for the base entry: an unmet capability warns + // rather than refusing, because the base always installs. + Capability string `json:"capability,omitempty" yaml:"capability,omitempty"` + // MinVRAM is this entry's own VRAM floor, in the same form as a + // candidate's (e.g. "2GiB"). It positions the entry within its own + // candidate list and is likewise advisory: falling short only warns. + MinVRAM string `json:"min_vram,omitempty" yaml:"min_vram,omitempty"` } func (m *GalleryModel) GetInstalled() bool { @@ -49,9 +58,11 @@ func (m GalleryModel) ID() string { return fmt.Sprintf("%s@%s", m.Gallery.Name, m.Name) } -// IsMeta reports whether this entry resolves to one of several concrete -// entries based on host hardware, rather than describing files directly. -func (m GalleryModel) IsMeta() bool { +// HasCandidates reports whether this entry declares hardware-gated upgrades +// over its own payload. It says nothing about the entry being installable: +// an entry with candidates is a complete entry that can always be installed +// as-is. +func (m GalleryModel) HasCandidates() bool { return len(m.Candidates) > 0 } diff --git a/core/schema/gallery-model.schema.json b/core/schema/gallery-model.schema.json index d43b42e07ad5..d72004fbd89e 100644 --- a/core/schema/gallery-model.schema.json +++ b/core/schema/gallery-model.schema.json @@ -56,9 +56,17 @@ "additionalProperties": false } }, + "capability": { + "type": "string", + "description": "Host capability this entry's own payload prefers, matched exactly and case-sensitively (for example metal, nvidia-cuda-12). Advisory: an unmet capability warns, it never blocks the install." + }, + "min_vram": { + "type": "string", + "description": "This entry's own VRAM floor, for example 2GiB. Positions the entry among its own candidates and must be strictly below every candidate's floor. Advisory: falling short only warns." + }, "candidates": { "type": "array", - "description": "Ordered variant list of a meta entry. The first candidate the host satisfies is installed, under the meta entry's own name. An entry carrying candidates must not also carry files or config_file.", + "description": "Ordered list of hardware-gated upgrades over this entry. The first candidate the host satisfies is installed, under this entry's own name; if none does, this entry installs itself. Clients that predate candidates support drop this key and install the entry unchanged.", "minItems": 1, "items": { "type": "object", @@ -66,7 +74,7 @@ "properties": { "model": { "type": "string", - "description": "Name of a concrete (non-meta) gallery entry" + "description": "Name of a gallery entry that declares no candidates of its own" }, "capability": { "type": "string", diff --git a/gallery/index.yaml b/gallery/index.yaml index c449afa6e445..57ebc707c633 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -4234,35 +4234,6 @@ - filename: llama-cpp/models/Qwen_Qwen3-Next-80B-A3B-Thinking-Q4_K_M.gguf sha256: 83481c75cc6c0837ba9afa52b59b4cd3f85f55dd7aa6c60e27230ff329c81367 uri: https://huggingface.co/bartowski/Qwen_Qwen3-Next-80B-A3B-Thinking-GGUF/resolve/main/Qwen_Qwen3-Next-80B-A3B-Thinking-Q4_K_M.gguf -- name: nanbeige4.1-3b - description: | - Nanbeige4.1-3B is built upon Nanbeige4-3B-Base and represents an enhanced iteration of our previous reasoning model, Nanbeige4-3B-Thinking-2511, achieved through further post-training optimization with supervised fine-tuning (SFT) and reinforcement learning (RL). As a highly competitive open-source model at a small parameter scale, Nanbeige4.1-3B illustrates that compact models can simultaneously achieve robust reasoning, preference alignment, and effective agentic behaviors. - - Automatically resolves to the best variant for your hardware: the Q8_0 build where there is enough VRAM for it, the Q4_K_M build otherwise. - license: apache-2.0 - icon: https://cdn-avatars.huggingface.co/v1/production/uploads/646f0d118ff94af23bc44aab/GXHCollpMRgvYqUXQ2BQ7.png - urls: - - https://huggingface.co/Nanbeige/Nanbeige4.1-3B - tags: - - nanbeige - - 3b - - llm - - gguf - - quantized - - chat - - reasoning - - agent - - multilingual - - instruction-tuned - # url is the fallback for LocalAI releases that predate candidates support: - # they drop the candidates key silently and would otherwise install nothing. - # Lint requires it to equal the final candidate's url exactly, so old and new - # clients agree on the least demanding option. - url: github:mudler/LocalAI/gallery/nanbeige4.1.yaml@master - candidates: - - model: nanbeige4.1-3b-q8 - min_vram: 6GiB - - model: nanbeige4.1-3b-q4 - name: nanbeige4.1-3b-q8 url: github:mudler/LocalAI/gallery/nanbeige4.1.yaml@master urls: @@ -4324,6 +4295,14 @@ - code - math last_checked: "2026-04-30" + # This entry installs the Q4_K_M build as-is, on any client and any host. + # min_vram positions it as its own last-resort candidate, and candidates + # lists the upgrades a bigger host gets instead. Clients that predate + # candidates support drop both keys and install this entry unchanged. + min_vram: 2GiB + candidates: + - model: nanbeige4.1-3b-q8 + min_vram: 6GiB overrides: parameters: model: nanbeige4.1-3b-q4_k_m.gguf From d7eb059f448937b7bc0d5b5d387a6e23af3353ea Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 22:05:27 +0000 Subject: [PATCH 16/48] feat(gallery): select model variants by hardware fit, not authored order Gallery entries could already carry a list of alternatives, but selection was an authored, ordered, first-match policy: every candidate declared a `capability` string and the VRAM floors had to descend in a hand-tuned order. That pushed hardware knowledge onto whoever edits the gallery and made ordering load-bearing, so a reordered list silently changed what users installed. None of it was necessary. SystemState.IsBackendCompatible already derives hardware support from a backend name alone: it knows MLX and metal are Darwin-only, CUDA is NVIDIA-only, ROCm AMD-only, SYCL Intel-only. Selection can read that instead of asking authors to restate it. Authoring is now just a list of names: - name: qwen3.6-27b min_memory: 4GiB variants: - model: qwen3.6-27b-mlx-8bit - model: qwen3.6-27b-gguf-q8 min_memory: 28GiB and all the intelligence moved into the selector. Given a host it drops the variants whose backend cannot run here, drops those whose known memory requirement exceeds what the host has, and takes the LARGEST of what is left, because a bigger footprint is a higher quality quantization of the same model. A variant of unknown size is kept, since nothing proves it does not fit, but it ranks last so a proven fit always beats a guess. An explicit pin still wins outright, and if nothing survives the entry installs its own payload: the base always installs, this never refuses. Available memory is VRAM when a GPU was detected and system RAM otherwise, read through xsysinfo so a cgroup limit is honored and a container gets its own limit rather than the node's RAM. Capability disappears entirely, from the types, the schema and the lint. VRAM and RAM collapse into one `min_memory`, because a model's footprint is roughly the same wherever it lives and one figure is compared against whichever applies. The lint rules about ordering, the capability vocabulary and floor relationships are deleted with the hazards they described; what remains is that every variant names an entry that exists and does not itself declare variants, plus that any memory figure actually parses. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- .github/ci/gallery_denormalize.go | 75 ++- core/gallery/candidate.go | 49 -- core/gallery/candidates_lint_test.go | 544 ------------------ core/gallery/model_artifacts.go | 8 +- core/gallery/models.go | 176 +++--- core/gallery/models_types.go | 38 +- core/gallery/resolve_candidate.go | 84 --- core/gallery/resolve_candidate_test.go | 108 ---- core/gallery/resolve_variant.go | 162 ++++++ core/gallery/resolve_variant_test.go | 273 +++++++++ core/gallery/variant.go | 56 ++ ...stall_test.go => variants_install_test.go} | 268 +++++---- core/gallery/variants_lint_test.go | 281 +++++++++ core/schema/gallery-model.schema.json | 26 +- gallery/index.yaml | 13 +- 15 files changed, 1113 insertions(+), 1048 deletions(-) delete mode 100644 core/gallery/candidate.go delete mode 100644 core/gallery/candidates_lint_test.go delete mode 100644 core/gallery/resolve_candidate.go delete mode 100644 core/gallery/resolve_candidate_test.go create mode 100644 core/gallery/resolve_variant.go create mode 100644 core/gallery/resolve_variant_test.go create mode 100644 core/gallery/variant.go rename core/gallery/{candidates_install_test.go => variants_install_test.go} (61%) create mode 100644 core/gallery/variants_lint_test.go diff --git a/.github/ci/gallery_denormalize.go b/.github/ci/gallery_denormalize.go index 0ae528f4705e..2dd0df909aa2 100644 --- a/.github/ci/gallery_denormalize.go +++ b/.github/ci/gallery_denormalize.go @@ -1,8 +1,8 @@ // Command gallery_denormalize fills the read-only denormalized fields on the -// candidates of gallery model entries: backend, quantization, and -// inferred_min_vram. +// variants of gallery model entries: backend, quantization, and +// inferred_min_memory. // -// It never modifies an authored min_vram. Authored values are authoritative, +// It never modifies an authored min_memory. Authored values are authoritative, // because a human who measured a real load knows more than a pre-download // estimate does. package main @@ -51,20 +51,20 @@ func main() { failures := 0 for i := range entries { - if !entries[i].HasCandidates() { + if !entries[i].HasVariants() { continue } - for j := range entries[i].Candidates { - c := &entries[i].Candidates[j] + for j := range entries[i].Variants { + c := &entries[i].Variants[j] target, ok := byName[c.Model] if !ok { - fmt.Fprintf(os.Stderr, "%s: candidate %q not found\n", entries[i].Name, c.Model) + fmt.Fprintf(os.Stderr, "%s: variant %q not found\n", entries[i].Name, c.Model) failures++ continue } - // The concrete entry's payload usually lives behind its url + // The referenced entry's payload usually lives behind its url // rather than inline, so fetch it. Metadata.Backend is populated // at load time by the running server, not by parsing the index, // so it is always empty here and cannot be copied. @@ -78,18 +78,17 @@ func main() { c.Backend = backendOf(target, resolved) c.Quantization = quantizationOf(target) - // Authored values win, and the final unconstrained candidate is - // deliberately floorless, so neither gets an inferred value. - // Clearing first matters: a candidate that gained an authored - // min_vram, or that became the last resort after a reorder, would - // otherwise keep a stale inferred floor that makes - // EffectiveMinVRAM report a constraint the entry no longer has. - if c.MinVRAM != "" || j == len(entries[i].Candidates)-1 { - c.InferredMinVRAM = "" + // An authored value wins outright, so it gets no inferred + // counterpart. Clearing first matters: a variant that gained an + // authored min_memory would otherwise keep a stale inferred figure + // that EffectiveMinMemory would go on reporting once the authored + // one is removed again. + if c.MinMemory != "" { + c.InferredMinMemory = "" continue } - // Weight files can come from either side: a concrete index entry + // Weight files can come from either side: a referenced index entry // usually lists them itself (files:, i.e. AdditionalFiles) while // the url it points at is only a base config, but some entries // invert that. Install time downloads both, so estimate over both. @@ -107,38 +106,38 @@ func main() { }, []uint32{estimateContext}) if err != nil || estimate.VRAMForContext(estimateContext) == 0 { fmt.Fprintf(os.Stderr, - "%s: could not estimate min_vram for %q (%v); author one by hand\n", + "%s: could not estimate min_memory for %q (%v); author one by hand\n", entries[i].Name, c.Model, err) failures++ continue } - c.InferredMinVRAM = fmt.Sprintf("%dMiB", estimate.VRAMForContext(estimateContext)/(1024*1024)) + c.InferredMinMemory = fmt.Sprintf("%dMiB", estimate.VRAMForContext(estimateContext)/(1024*1024)) } } - if err := writeCandidates(path, data, entries); err != nil { + if err := writeVariants(path, data, entries); err != nil { fmt.Fprintf(os.Stderr, "write %s: %v\n", path, err) os.Exit(1) } if failures > 0 { - fmt.Fprintf(os.Stderr, "%d candidate(s) need a hand-authored min_vram\n", failures) + fmt.Fprintf(os.Stderr, "%d variant(s) need a hand-authored min_memory\n", failures) os.Exit(1) } } -// writeCandidates writes back only the three derived keys on each candidate, +// writeVariants writes back only the three derived keys on each variant, // leaving every other byte of the index alone. // // Round-tripping through []GalleryModel would reflow all 1200+ entries // (quoting, key order, line wrapping), burying the handful of real changes in // reformatting noise and making the nightly PR unreviewable. Even re-encoding -// just the candidates subtree would restyle authored fields such as min_vram, +// just the variants subtree would restyle authored fields such as min_memory, // so the rewrite reaches down to individual mapping keys and the node tree is // re-encoded at the index's authored indent. That also makes the job // idempotent: a run that computes the same values leaves the file untouched // and opens no PR. -func writeCandidates(path string, data []byte, entries []gallery.GalleryModel) error { +func writeVariants(path string, data []byte, entries []gallery.GalleryModel) error { var doc yaml.Node if err := yaml.Unmarshal(data, &doc); err != nil { return fmt.Errorf("re-parse for node rewrite: %w", err) @@ -156,30 +155,30 @@ func writeCandidates(path string, data []byte, entries []gallery.GalleryModel) e changed := false for i, entryNode := range root.Content { - if !entries[i].HasCandidates() || entryNode.Kind != yaml.MappingNode { + if !entries[i].HasVariants() || entryNode.Kind != yaml.MappingNode { continue } - candidatesNode := mappingValue(entryNode, "candidates") - if candidatesNode == nil || candidatesNode.Kind != yaml.SequenceNode { - return fmt.Errorf("entry %q has candidates but no candidates sequence in the document", entries[i].Name) + variantsNode := mappingValue(entryNode, "variants") + if variantsNode == nil || variantsNode.Kind != yaml.SequenceNode { + return fmt.Errorf("entry %q has variants but no variants sequence in the document", entries[i].Name) } - if len(candidatesNode.Content) != len(entries[i].Candidates) { - return fmt.Errorf("entry %q candidate count drift: %d nodes vs %d parsed", - entries[i].Name, len(candidatesNode.Content), len(entries[i].Candidates)) + if len(variantsNode.Content) != len(entries[i].Variants) { + return fmt.Errorf("entry %q variant count drift: %d nodes vs %d parsed", + entries[i].Name, len(variantsNode.Content), len(entries[i].Variants)) } - for j, candidateNode := range candidatesNode.Content { - if candidateNode.Kind != yaml.MappingNode { - return fmt.Errorf("entry %q candidate %d is not a mapping", entries[i].Name, j) + for j, variantNode := range variantsNode.Content { + if variantNode.Kind != yaml.MappingNode { + return fmt.Errorf("entry %q variant %d is not a mapping", entries[i].Name, j) } - c := entries[i].Candidates[j] + c := entries[i].Variants[j] for _, kv := range []struct{ key, value string }{ {"backend", c.Backend}, {"quantization", c.Quantization}, - {"inferred_min_vram", c.InferredMinVRAM}, + {"inferred_min_memory", c.InferredMinMemory}, } { - if setMappingValue(candidateNode, kv.key, kv.value) { + if setMappingValue(variantNode, kv.key, kv.value) { changed = true } } @@ -287,7 +286,7 @@ func backendOf(m gallery.GalleryModel, resolved gallery.ModelConfig) string { return "" } -// quantizationOf reports the quantization declared by a concrete entry's +// quantizationOf reports the quantization declared by a referenced entry's // overrides, empty when it declares none. func quantizationOf(m gallery.GalleryModel) string { if q, ok := m.Overrides["quantization"].(string); ok { diff --git a/core/gallery/candidate.go b/core/gallery/candidate.go deleted file mode 100644 index 9e5b342981a3..000000000000 --- a/core/gallery/candidate.go +++ /dev/null @@ -1,49 +0,0 @@ -package gallery - -import ( - "fmt" - - "github.com/mudler/LocalAI/pkg/vram" -) - -// Candidate is one option in a gallery entry's ordered candidate list. It -// references an existing gallery entry by name and declares the conditions -// under which that entry is the right choice for the host. -type Candidate struct { - // Model is the name of a gallery entry that declares no candidates of its - // own, or the name of the declaring entry itself when it stands as its own - // last-resort candidate. - Model string `json:"model" yaml:"model"` - // Capability, when set, must equal the host's reported capability - // (e.g. "metal", "nvidia-cuda-12"). Empty matches any host. - Capability string `json:"capability,omitempty" yaml:"capability,omitempty"` - // MinVRAM is the authored VRAM floor (e.g. "20GiB"). Authoritative: - // inference never overwrites it. - MinVRAM string `json:"min_vram,omitempty" yaml:"min_vram,omitempty"` - - // The fields below are denormalized by the nightly job for display and - // lint. They are never authored by hand and never affect what gets - // installed, because installation reads the referenced entry live. - Backend string `json:"backend,omitempty" yaml:"backend,omitempty"` - Quantization string `json:"quantization,omitempty" yaml:"quantization,omitempty"` - InferredMinVRAM string `json:"inferred_min_vram,omitempty" yaml:"inferred_min_vram,omitempty"` -} - -// EffectiveMinVRAM returns the VRAM floor in bytes for this candidate and -// whether a floor is declared at all. An authored MinVRAM wins over an -// inferred one. A candidate with no floor declares no requirement and always -// passes the VRAM check. -func (c Candidate) EffectiveMinVRAM() (uint64, bool, error) { - raw := c.MinVRAM - if raw == "" { - raw = c.InferredMinVRAM - } - if raw == "" { - return 0, false, nil - } - bytes, err := vram.ParseSizeString(raw) - if err != nil { - return 0, false, fmt.Errorf("candidate %q has an unparseable min_vram %q: %w", c.Model, raw, err) - } - return bytes, true, nil -} diff --git a/core/gallery/candidates_lint_test.go b/core/gallery/candidates_lint_test.go deleted file mode 100644 index 5ba9dd52d4a5..000000000000 --- a/core/gallery/candidates_lint_test.go +++ /dev/null @@ -1,544 +0,0 @@ -package gallery_test - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "sync" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "gopkg.in/yaml.v3" - - "github.com/mudler/LocalAI/core/gallery" -) - -// knownCapabilities mirrors the values SystemState.DetectedCapability() can -// actually report, which is what the candidate resolver compares against with -// a case-sensitive exact match. A capability outside this set can never match, -// so a typo would silently make a candidate unreachable on every host. -// -// Note "cpu" is deliberately absent: it exists only as a fallback key inside -// SystemState.Capability(capMap) for meta backends, and is never a value -// getSystemCapabilities() returns. A CPU-only host reports "default". -var knownCapabilities = map[string]bool{ - "default": true, "metal": true, "darwin-x86": true, - "nvidia": true, "nvidia-cuda-12": true, "nvidia-cuda-13": true, - "nvidia-l4t": true, "nvidia-l4t-cuda-12": true, "nvidia-l4t-cuda-13": true, - "intel": true, "amd": true, "vulkan": true, -} - -// candidateViolation is one invariant breach found by a lint helper. Helpers -// return every breach they find instead of stopping at the first, so a single -// run names them all rather than forcing a fix-one-rerun-repeat cycle. -type candidateViolation struct { - Entry string - Candidate string - Detail string -} - -func (v candidateViolation) String() string { - if v.Candidate == "" { - return fmt.Sprintf("%s: %s", v.Entry, v.Detail) - } - return fmt.Sprintf("%s -> candidate %q: %s", v.Entry, v.Candidate, v.Detail) -} - -func formatViolations(violations []candidateViolation) string { - lines := make([]string, 0, len(violations)) - for _, v := range violations { - lines = append(lines, " "+v.String()) - } - return "\n" + strings.Join(lines, "\n") -} - -func indexEntriesByName(entries []gallery.GalleryModel) map[string]gallery.GalleryModel { - byName := make(map[string]gallery.GalleryModel, len(entries)) - for _, e := range entries { - byName[e.Name] = e - } - return byName -} - -// baseCandidate mirrors the implicit last-resort candidate the resolver -// synthesizes from the entry itself, so lint measures the same floor the -// installer will. -func baseCandidate(e gallery.GalleryModel) gallery.Candidate { - return gallery.Candidate{Model: e.Name, Capability: e.Capability, MinVRAM: e.MinVRAM} -} - -// checkCandidateReferences verifies every candidate names an existing entry -// that declares no candidates of its own. Resolution is a single pass, so a -// nested reference would silently ignore the inner list. -func checkCandidateReferences(entries []gallery.GalleryModel) []candidateViolation { - byName := indexEntriesByName(entries) - var violations []candidateViolation - for _, e := range entries { - if !e.HasCandidates() { - continue - } - for _, c := range e.Candidates { - target, ok := byName[c.Model] - if !ok { - violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "references unknown model"}) - continue - } - if target.HasCandidates() { - violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "references an entry that declares candidates of its own; nesting is not allowed"}) - } - } - } - return violations -} - -// checkCandidateFloors verifies every declared candidate carries a parseable -// VRAM floor. A candidate is an UPGRADE over the entry, so it must say what it -// costs; a floorless one would capture every host and make the entry's own -// payload unreachable. -func checkCandidateFloors(entries []gallery.GalleryModel) []candidateViolation { - var violations []candidateViolation - for _, e := range entries { - if !e.HasCandidates() { - continue - } - for _, c := range e.Candidates { - _, declared, err := c.EffectiveMinVRAM() - switch { - case err != nil: - violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "has a bad min_vram: " + err.Error()}) - case !declared: - violations = append(violations, candidateViolation{Entry: e.Name, Candidate: c.Model, Detail: "needs a min_vram; the nightly job should have inferred one"}) - } - } - } - return violations -} - -// checkBaseFloor verifies the entry's own floor sits strictly below every -// candidate's. The entry is the last element of its own candidate list, so a -// base floor at or above a candidate's makes that candidate dead: any host -// clearing it already cleared the base, and first-match never gets that far. -func checkBaseFloor(entries []gallery.GalleryModel) []candidateViolation { - var violations []candidateViolation - for _, e := range entries { - if !e.HasCandidates() { - continue - } - base := baseCandidate(e) - baseFloor, _, err := base.EffectiveMinVRAM() - if err != nil { - violations = append(violations, candidateViolation{Entry: e.Name, Detail: "has a bad min_vram: " + err.Error()}) - continue - } - for _, c := range e.Candidates { - floor, declared, cerr := c.EffectiveMinVRAM() - if cerr != nil || !declared { - // checkCandidateFloors owns reporting those. - continue - } - if baseFloor >= floor { - violations = append(violations, candidateViolation{ - Entry: e.Name, - Candidate: c.Model, - Detail: fmt.Sprintf("sits at or below the entry's own %d byte floor, so it can never be reached", - baseFloor), - }) - } - } - } - return violations -} - -// checkCandidateCapabilities verifies entries and candidates only name -// capabilities the system can actually report, since the resolver compares -// them exactly. -func checkCandidateCapabilities(entries []gallery.GalleryModel) []candidateViolation { - var violations []candidateViolation - for _, e := range entries { - if !e.HasCandidates() { - continue - } - if e.Capability != "" && !knownCapabilities[e.Capability] { - violations = append(violations, candidateViolation{ - Entry: e.Name, - Detail: fmt.Sprintf("uses unknown capability %q", e.Capability), - }) - } - for _, c := range e.Candidates { - if c.Capability == "" { - continue - } - if !knownCapabilities[c.Capability] { - violations = append(violations, candidateViolation{ - Entry: e.Name, - Candidate: c.Model, - Detail: fmt.Sprintf("uses unknown capability %q", c.Capability), - }) - } - } - } - return violations -} - -// checkCandidateOrdering verifies no candidate is shadowed by an earlier one. -// Selection is first-match over the authored order, so a candidate is dead -// whenever an earlier one matches every host this one would. -// -// Two shapes cause that. Within a single capability group the groups are -// mutually exclusive, so only a floor rising above an earlier floor is -// unreachable. A candidate with an EMPTY capability matches every host and so -// dominates ACROSS groups: every later candidate whose floor is at or above -// the running minimum unconditional floor is dead, whatever capability it asks -// for. Tracking that running minimum subsumes the same-group check for the -// empty capability. -func checkCandidateOrdering(entries []gallery.GalleryModel) []candidateViolation { - var violations []candidateViolation - for _, e := range entries { - if !e.HasCandidates() { - continue - } - previous := map[string]uint64{} - var unconditionalFloor uint64 - haveUnconditional := false - - for _, c := range e.Candidates { - floor, declared, err := c.EffectiveMinVRAM() - if err != nil || !declared { - // Parse errors and absent floors belong to checkCandidateFloors. - continue - } - - switch { - case haveUnconditional && floor >= unconditionalFloor: - violations = append(violations, candidateViolation{ - Entry: e.Name, - Candidate: c.Model, - Detail: fmt.Sprintf("is shadowed by an earlier candidate that matches any host at a %d byte floor, so it can never be reached", - unconditionalFloor), - }) - case c.Capability != "": - if prior, seen := previous[c.Capability]; seen && floor > prior { - violations = append(violations, candidateViolation{ - Entry: e.Name, - Candidate: c.Model, - Detail: "raises the VRAM floor after a lower one in the same capability group, so it can never be reached", - }) - } - } - - if c.Capability == "" && (!haveUnconditional || floor < unconditionalFloor) { - unconditionalFloor = floor - haveUnconditional = true - } - previous[c.Capability] = floor - } - } - return violations -} - -// loadGalleryIndex parses gallery/index.yaml once for the whole suite. The -// index carries well over a thousand entries, so re-parsing it per spec is -// pure overhead. -var loadGalleryIndex = sync.OnceValues(func() ([]gallery.GalleryModel, error) { - data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) - if err != nil { - return nil, err - } - var entries []gallery.GalleryModel - if err := yaml.Unmarshal(data, &entries); err != nil { - return nil, err - } - return entries, nil -}) - -func plainEntry(name, url string) gallery.GalleryModel { - e := gallery.GalleryModel{} - e.Name = name - e.URL = url - return e -} - -func entryWithCandidates(name, url, minVRAM string, candidates ...gallery.Candidate) gallery.GalleryModel { - e := gallery.GalleryModel{Candidates: candidates, MinVRAM: minVRAM} - e.Name = name - e.URL = url - return e -} - -// candidateFixture builds an entry with candidates plus the entries it -// references. Synthetic fixtures keep the invariant logic covered however few -// entries in gallery/index.yaml declare candidates; without them every -// index-driven spec below is a no-op that passes while checking nothing. -func candidateFixture(base gallery.GalleryModel, referenced ...gallery.GalleryModel) []gallery.GalleryModel { - return append([]gallery.GalleryModel{base}, referenced...) -} - -var _ = Describe("gallery candidate lint helpers", func() { - // An entry declares candidates solely by carrying the key. GalleryBackend.IsMeta() - // has deliberately different semantics (it requires an EMPTY uri), so a - // well-meaning alignment of the two would make every helper skip every - // entry and pass silently. Assert the distinction directly. - It("treats an entry with candidates as such even though it has a url", func() { - Expect(entryWithCandidates("base", "u://base", "2GiB", gallery.Candidate{Model: "big"}).HasCandidates()).To(BeTrue()) - Expect(plainEntry("big", "u://big").HasCandidates()).To(BeFalse()) - }) - - It("passes every invariant on a valid entry", func() { - entries := candidateFixture( - entryWithCandidates("base", "u://base", "4GiB", - gallery.Candidate{Model: "big", Capability: "nvidia", MinVRAM: "24GiB"}, - gallery.Candidate{Model: "mid", Capability: "nvidia", MinVRAM: "12GiB"}, - gallery.Candidate{Model: "metal-big", Capability: "metal", MinVRAM: "32GiB"}, - gallery.Candidate{Model: "small", MinVRAM: "8GiB"}, - ), - plainEntry("big", "u://big"), - plainEntry("mid", "u://mid"), - plainEntry("metal-big", "u://metal-big"), - plainEntry("small", "u://small"), - ) - - Expect(checkCandidateReferences(entries)).To(BeEmpty()) - Expect(checkCandidateFloors(entries)).To(BeEmpty()) - Expect(checkBaseFloor(entries)).To(BeEmpty()) - Expect(checkCandidateCapabilities(entries)).To(BeEmpty()) - Expect(checkCandidateOrdering(entries)).To(BeEmpty()) - }) - - It("passes every invariant on an entry that declares no floor of its own", func() { - // A floorless entry is the weakest possible base, below every - // candidate, which is exactly the position the base should hold. - entries := candidateFixture( - entryWithCandidates("base", "u://base", "", - gallery.Candidate{Model: "big", MinVRAM: "24GiB"}, - ), - plainEntry("big", "u://big"), - ) - - Expect(checkCandidateFloors(entries)).To(BeEmpty()) - Expect(checkBaseFloor(entries)).To(BeEmpty()) - Expect(checkCandidateOrdering(entries)).To(BeEmpty()) - }) - - Describe("checkBaseFloor", func() { - It("flags a candidate whose floor sits below the entry's own", func() { - entries := candidateFixture( - entryWithCandidates("base", "u://base", "20GiB", - gallery.Candidate{Model: "big", MinVRAM: "8GiB"}, - ), - plainEntry("big", "u://big"), - ) - - violations := checkBaseFloor(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("big")) - Expect(violations[0].Detail).To(ContainSubstring("at or below the entry's own")) - }) - - It("flags a candidate whose floor merely equals the entry's own", func() { - // Equal floors make the candidate unreachable just as surely: every - // host clearing it clears the base, and the base is checked first - // only in the sense that first-match never reaches this candidate. - entries := candidateFixture( - entryWithCandidates("base", "u://base", "8GiB", - gallery.Candidate{Model: "big", MinVRAM: "8GiB"}, - ), - plainEntry("big", "u://big"), - ) - - violations := checkBaseFloor(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("big")) - }) - - It("flags an entry whose own min_vram cannot be parsed", func() { - entries := candidateFixture( - entryWithCandidates("base", "u://base", "eight gigs", - gallery.Candidate{Model: "big", MinVRAM: "24GiB"}, - ), - plainEntry("big", "u://big"), - ) - - violations := checkBaseFloor(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Entry).To(Equal("base")) - Expect(violations[0].Candidate).To(BeEmpty()) - Expect(violations[0].Detail).To(ContainSubstring("bad min_vram")) - }) - }) - - Describe("checkCandidateOrdering", func() { - It("flags a candidate shadowed by an earlier unconditional candidate", func() { - // "a" matches any host at 8GiB, so the nvidia candidate at 20GiB is - // dead: every nvidia host clearing 20GiB already cleared 8GiB. - entries := candidateFixture( - entryWithCandidates("base", "u://base", "2GiB", - gallery.Candidate{Model: "a", MinVRAM: "8GiB"}, - gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"}, - ), - plainEntry("a", "u://a"), plainEntry("b", "u://b"), - ) - - violations := checkCandidateOrdering(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("b")) - Expect(violations[0].Detail).To(ContainSubstring("shadowed by an earlier candidate that matches any host")) - }) - - It("flags a floor inversion inside one capability group", func() { - entries := candidateFixture( - entryWithCandidates("base", "u://base", "2GiB", - gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"}, - gallery.Candidate{Model: "b", Capability: "nvidia", MinVRAM: "20GiB"}, - ), - plainEntry("a", "u://a"), plainEntry("b", "u://b"), - ) - - violations := checkCandidateOrdering(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("b")) - Expect(violations[0].Detail).To(ContainSubstring("same capability group")) - }) - - It("keeps distinct capability groups independent", func() { - // A high metal floor after a low nvidia floor is fine: no host - // reports both capabilities. - entries := candidateFixture( - entryWithCandidates("base", "u://base", "2GiB", - gallery.Candidate{Model: "a", Capability: "nvidia", MinVRAM: "8GiB"}, - gallery.Candidate{Model: "b", Capability: "metal", MinVRAM: "32GiB"}, - ), - plainEntry("a", "u://a"), plainEntry("b", "u://b"), - ) - - Expect(checkCandidateOrdering(entries)).To(BeEmpty()) - }) - }) - - Describe("checkCandidateFloors", func() { - It("flags a candidate with no floor", func() { - entries := candidateFixture( - entryWithCandidates("base", "u://base", "2GiB", - gallery.Candidate{Model: "a", Capability: "nvidia"}, - ), - plainEntry("a", "u://a"), - ) - - violations := checkCandidateFloors(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("a")) - Expect(violations[0].Detail).To(ContainSubstring("needs a min_vram")) - }) - - It("flags a candidate whose floor cannot be parsed", func() { - entries := candidateFixture( - entryWithCandidates("base", "u://base", "2GiB", - gallery.Candidate{Model: "a", MinVRAM: "lots"}, - ), - plainEntry("a", "u://a"), - ) - - violations := checkCandidateFloors(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("a")) - Expect(violations[0].Detail).To(ContainSubstring("bad min_vram")) - }) - }) - - Describe("checkCandidateReferences", func() { - It("flags a candidate naming an entry that does not exist", func() { - entries := candidateFixture( - entryWithCandidates("base", "u://base", "2GiB", - gallery.Candidate{Model: "ghost", MinVRAM: "20GiB"}, - ), - plainEntry("a", "u://a"), - ) - - violations := checkCandidateReferences(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("ghost")) - Expect(violations[0].Detail).To(ContainSubstring("unknown model")) - }) - - It("flags a candidate naming an entry that declares candidates itself", func() { - entries := candidateFixture( - entryWithCandidates("base", "u://base", "2GiB", - gallery.Candidate{Model: "nested", MinVRAM: "20GiB"}, - ), - entryWithCandidates("nested", "u://nested", "4GiB", gallery.Candidate{Model: "a", MinVRAM: "30GiB"}), - plainEntry("a", "u://a"), - ) - - violations := checkCandidateReferences(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("nested")) - Expect(violations[0].Detail).To(ContainSubstring("nesting is not allowed")) - }) - }) - - Describe("checkCandidateCapabilities", func() { - It("flags a capability the system can never report", func() { - entries := candidateFixture( - entryWithCandidates("base", "u://base", "2GiB", - gallery.Candidate{Model: "a", Capability: "cuda", MinVRAM: "20GiB"}, - ), - plainEntry("a", "u://a"), - ) - - violations := checkCandidateCapabilities(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Candidate).To(Equal("a")) - Expect(violations[0].Detail).To(ContainSubstring(`unknown capability "cuda"`)) - }) - - It("flags an unknown capability on the entry itself", func() { - entry := entryWithCandidates("base", "u://base", "2GiB", gallery.Candidate{Model: "a", MinVRAM: "20GiB"}) - entry.Capability = "NVIDIA" - entries := candidateFixture(entry, plainEntry("a", "u://a")) - - violations := checkCandidateCapabilities(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Entry).To(Equal("base")) - Expect(violations[0].Candidate).To(BeEmpty()) - Expect(violations[0].Detail).To(ContainSubstring(`unknown capability "NVIDIA"`)) - }) - }) -}) - -var _ = Describe("gallery/index.yaml candidate invariants", Ordered, func() { - var entries []gallery.GalleryModel - - BeforeAll(func() { - var err error - entries, err = loadGalleryIndex() - Expect(err).ToNot(HaveOccurred()) - // A truncated or emptied index unmarshals cleanly and would make every - // spec below vacuously pass. - Expect(entries).ToNot(BeEmpty()) - }) - - It("references only existing entries that declare no candidates themselves", func() { - v := checkCandidateReferences(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) - - It("gives every candidate a VRAM floor", func() { - v := checkCandidateFloors(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) - - It("keeps every entry's own floor below its candidates'", func() { - v := checkBaseFloor(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) - - It("uses only capabilities the system can report", func() { - v := checkCandidateCapabilities(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) - - It("orders candidates so no candidate is shadowed by an earlier one", func() { - v := checkCandidateOrdering(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) -}) diff --git a/core/gallery/model_artifacts.go b/core/gallery/model_artifacts.go index d323293e6ba0..b68cbab17f4b 100644 --- a/core/gallery/model_artifacts.go +++ b/core/gallery/model_artifacts.go @@ -32,10 +32,10 @@ func WithArtifactMaterializer(materializer ArtifactMaterializer) InstallOption { } } -// WithVariant pins a gallery entry to a specific candidate by name, bypassing -// hardware-based resolution. The entry's own name is a valid pin, since the -// entry is itself the last-resort candidate. Ignored for entries that declare -// no candidates. +// WithVariant pins a gallery entry to a specific variant by name, bypassing +// hardware-based selection. The entry's own name is a valid pin, since the +// entry is itself the last resort. Ignored for entries that declare no +// variants. func WithVariant(variant string) InstallOption { return func(options *installOptions) { options.variant = variant diff --git a/core/gallery/models.go b/core/gallery/models.go index 917e3861826f..d8bf109c1868 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -17,6 +17,7 @@ import ( "github.com/mudler/LocalAI/pkg/modelartifacts" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" + "github.com/mudler/LocalAI/pkg/xsysinfo" "github.com/mudler/xlog" "gopkg.in/yaml.v3" @@ -61,7 +62,7 @@ type ModelConfig struct { Files []File `yaml:"files"` PromptTemplates []PromptTemplate `yaml:"prompt_templates"` - // The fields below record how an entry carrying candidates was resolved, + // The fields below record how an entry carrying variants was resolved, // so a reinstall or upgrade can honor the same pin and so operators can // see which variant a stable model name is actually backed by. EntryName string `yaml:"entry_name,omitempty"` @@ -80,24 +81,38 @@ type PromptTemplate struct { Content string `yaml:"content"` } -// effectiveCandidates returns the candidate list resolution actually runs over: -// the declared upgrades followed by the entry itself. +// variantOptions turns an entry's declared variants into selectable options, +// resolving each referenced gallery entry so the selector gets the backend it +// filters on, and appends the entry itself as the base option. // -// The entry is appended rather than special-cased so there is exactly one -// selection path. Being last makes it the last resort, which is what keeps the -// entry always installable: the list can never be exhausted without reaching it. -func effectiveCandidates(entry *GalleryModel) []Candidate { - candidates := make([]Candidate, 0, len(entry.Candidates)+1) - candidates = append(candidates, entry.Candidates...) - return append(candidates, Candidate{ - Model: entry.Name, - Capability: entry.Capability, - MinVRAM: entry.MinVRAM, - }) +// The base is appended rather than special-cased downstream so there is exactly +// one selection path, and so an operator can pin the entry's own name to +// decline an upgrade their hardware would otherwise take. +func variantOptions(models []*GalleryModel, entry *GalleryModel) ([]VariantOption, error) { + options := make([]VariantOption, 0, len(entry.Variants)+1) + for _, v := range entry.Variants { + // Every referenced entry is looked up here rather than lazily after + // selection, because the backend that decides whether a variant is even + // eligible lives on that entry, not on the variant. + target := FindGalleryElement(models, v.Model) + if target == nil { + return nil, fmt.Errorf("model %q references variant %q which does not exist in any configured gallery", entry.Name, v.Model) + } + if target.HasVariants() { + return nil, fmt.Errorf("model %q references variant %q which declares variants of its own; resolution is a single pass, so those would be silently ignored", entry.Name, v.Model) + } + options = append(options, VariantOption{Variant: v, Backend: target.Backend}) + } + + return append(options, VariantOption{ + Variant: Variant{Model: entry.Name, MinMemory: entry.MinMemory}, + Backend: entry.Backend, + IsBase: true, + }), nil } // ResolveVariant picks the gallery entry to install for a host, from the -// upgrades an entry declares plus the entry itself, and returns it renamed to +// variants an entry declares plus the entry itself, and returns it renamed to // the entry's name and carrying the entry's presentation metadata. // // Why the metadata split: the payload (url, config_file, files, overrides) @@ -106,53 +121,44 @@ func effectiveCandidates(entry *GalleryModel) []Candidate { // from the entry the user asked for, so the installed model presents as that // model rather than as one of its variants. // -// The base entry always resolves. When even its own predicates fall short this -// warns and installs it regardless, because the entry is a complete entry that -// every older LocalAI release installs unconditionally; refusing it here would -// make the gallery behave worse the newer the client is. -func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, pin string) (*GalleryModel, Candidate, error) { - candidates := effectiveCandidates(entry) - base := candidates[len(candidates)-1] - - candidate, err := ResolveCandidate(candidates, env, pin) +// The base entry always resolves. When even its own requirement falls short +// this installs it regardless, because the entry is a complete entry that every +// older LocalAI release installs unconditionally; refusing it here would make +// the gallery behave worse the newer the client is. +func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, pin string) (*GalleryModel, Variant, error) { + options, err := variantOptions(models, entry) if err != nil { - if !errors.Is(err, ErrNoCandidateMatch) { - return nil, Candidate{}, fmt.Errorf("resolving variant for model %q: %w", entry.Name, err) - } - xlog.Warn("This system does not meet the requirements this model declares for itself; installing it anyway because it is the last resort", - "model", entry.Name, "capability", env.Capability, "available_vram", env.VRAM, "reason", err) - candidate = base + return nil, Variant{}, err + } + + selection, err := SelectVariant(options, env, pin) + if err != nil { + return nil, Variant{}, fmt.Errorf("resolving variant for model %q: %w", entry.Name, err) + } + selected := selection.Option + + if selection.FellBackToBase && len(entry.Variants) > 0 { + xlog.Warn("No declared variant of this model fits this system; installing the entry's own build", + "model", entry.Name, "available_memory", env.AvailableMemory, "reasons", strings.Join(selection.Reasons, "; ")) } // A pin is an operator override and deliberately bypasses the hardware // checks, but a silent bypass makes a later out-of-memory failure // impossible to trace back to the pin, so it is recorded loudly here. - // It is warned about only once the pin is known to name a real, installable - // entry, otherwise a pin naming a listed-but-nonexistent variant would warn - // about VRAM and then fail for an entirely unrelated reason. - warnPin := func() { - if pin == "" { - return - } - if floor, declared, verr := candidate.EffectiveMinVRAM(); verr == nil && declared && env.VRAM < floor { - xlog.Warn("Pinned model variant declares more VRAM than this system reports; installing anyway because the pin overrides hardware resolution", - "model", entry.Name, "variant", candidate.Model, "required_vram", floor, "available_vram", env.VRAM) + if pin != "" { + if need, known, verr := selected.Variant.EffectiveMinMemory(); verr == nil && known && env.AvailableMemory < need { + xlog.Warn("Pinned model variant declares more memory than this system reports; installing anyway because the pin overrides hardware resolution", + "model", entry.Name, "variant", selected.Variant.Model, "required_memory", need, "available_memory", env.AvailableMemory) } } - // Resolving to the base means installing the entry's own payload, which is - // the entry itself; there is no second entry to look up. + // Selecting the base means installing the entry's own payload, which is the + // entry itself; there is no second entry to look up. source := entry - if candidate.Model != entry.Name { - source = FindGalleryElement(models, candidate.Model) - if source == nil { - return nil, Candidate{}, fmt.Errorf("model %q references variant %q which does not exist in any configured gallery", entry.Name, candidate.Model) - } - if source.HasCandidates() { - return nil, Candidate{}, fmt.Errorf("model %q references variant %q which declares candidates of its own; resolution is a single pass, so those would be silently ignored", entry.Name, candidate.Model) - } + if !selected.IsBase { + // variantOptions already proved this lookup succeeds. + source = FindGalleryElement(models, selected.Variant.Model) } - warnPin() resolved := *source resolved.Name = entry.Name @@ -162,9 +168,8 @@ func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, // The resolved entry is a concrete install target, so it must not carry the // selection fields any more; leaving them would let a second pass resolve // the already-resolved entry all over again. - resolved.Candidates = nil - resolved.Capability = "" - resolved.MinVRAM = "" + resolved.Variants = nil + resolved.MinMemory = "" // The struct copy above is shallow, so every reference-typed field still // aliases the gallery's own entries. The install path mutates Overrides in @@ -185,7 +190,33 @@ func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, resolved.URLs = slices.Clone(entry.URLs) resolved.Tags = slices.Clone(entry.Tags) - return &resolved, candidate, nil + return &resolved, selected.Variant, nil +} + +// noGPUDetected is what SystemState.DetectedCapability() reports when no +// usable GPU was found. The constant is unexported in pkg/system. +const noGPUDetected = "default" + +// availableModelMemory reports how much memory a model may occupy on this host. +// +// With a usable GPU the model lives in VRAM. Without one it lives in system +// RAM, read through xsysinfo because that path is cgroup-aware: under +// Kubernetes the container's limit, not the node's physical RAM, is what the +// model actually gets. +// +// An unreadable RAM figure yields 0, which drops every variant with a known +// requirement and installs the base. That is the safe direction: an unknown +// host should not be talked into a larger download. +func availableModelMemory(systemState *system.SystemState) uint64 { + if systemState.DetectedCapability() != noGPUDetected { + return systemState.VRAM + } + ram, err := xsysinfo.GetSystemRAMInfo() + if err != nil || ram == nil { + xlog.Warn("Could not read system RAM; treating this host as having no memory budget for variant selection", "error", err) + return 0 + } + return ram.Total } // deepCopyStringMap copies a decoded YAML map so no part of the result, at any @@ -276,7 +307,7 @@ func InstallModelFromGallery( config.ResolvedVariant = record.ResolvedVariant config.PinnedVariant = record.PinnedVariant // The variant's own name would otherwise be persisted here, which - // contradicts the whole point of candidates: the model is known by + // contradicts the whole point of variants: the model is known by // the entry's stable name regardless of which variant backs it. config.Name = record.EntryName } @@ -325,10 +356,10 @@ func InstallModelFromGallery( return fmt.Errorf("no model found with name %q", name) } - // An entry without candidates is installed directly. An entry with them is - // still installable as-is; resolution below only decides whether one of its - // declared upgrades fits this host better. - if !model.HasCandidates() { + // An entry without variants is installed directly. An entry with them is + // still installable as-is; selection below only decides whether one of its + // declared alternatives suits this host better. + if !model.HasVariants() { return applyModel(model, nil) } @@ -354,22 +385,31 @@ func InstallModelFromGallery( } env := ResolveEnv{ - Capability: systemState.DetectedCapability(), - VRAM: systemState.VRAM, - } - - resolved, candidate, err := ResolveVariant(models, model, env, pin) + AvailableMemory: availableModelMemory(systemState), + // The whole hardware gate. IsBackendCompatible already derives + // Darwin-only, NVIDIA-only, ROCm-only and SYCL-only from the backend + // name, so a gallery author never has to describe hardware. + // + // The uri argument is deliberately empty: it exists for backend OCI + // images, and passing a model's gallery url here would let an unrelated + // substring in a download link decide hardware compatibility. + BackendCompatible: func(backend string) bool { + return systemState.IsBackendCompatible(backend, "") + }, + } + + resolved, variant, err := ResolveVariant(models, model, env, pin) if err != nil { return err } xlog.Info("Resolved model to variant", - "model", model.Name, "variant", candidate.Model, - "capability", env.Capability, "vram", env.VRAM, "pinned", pin != "") + "model", model.Name, "variant", variant.Model, + "available_memory", env.AvailableMemory, "pinned", pin != "") return applyModel(resolved, &ModelConfig{ EntryName: model.Name, - ResolvedVariant: candidate.Model, + ResolvedVariant: variant.Model, PinnedVariant: pin, }) } diff --git a/core/gallery/models_types.go b/core/gallery/models_types.go index 3b570d8cc02f..2a48ca68421c 100644 --- a/core/gallery/models_types.go +++ b/core/gallery/models_types.go @@ -15,19 +15,20 @@ type GalleryModel struct { ConfigFile map[string]any `json:"config_file,omitempty" yaml:"config_file,omitempty"` // Overrides are used to override the configuration of the model located at URL Overrides map[string]any `json:"overrides,omitempty" yaml:"overrides,omitempty"` - // Candidates is an optional ordered list of hardware-gated UPGRADES over - // this entry. The entry itself is always the last-resort candidate, so an - // entry carrying candidates stays a complete, installable entry and older - // LocalAI releases, which drop this key, install it exactly as before. - Candidates []Candidate `json:"candidates,omitempty" yaml:"candidates,omitempty"` - // Capability, when set, is the host capability this entry's own payload - // prefers. It is advisory for the base entry: an unmet capability warns - // rather than refusing, because the base always installs. - Capability string `json:"capability,omitempty" yaml:"capability,omitempty"` - // MinVRAM is this entry's own VRAM floor, in the same form as a - // candidate's (e.g. "2GiB"). It positions the entry within its own - // candidate list and is likewise advisory: falling short only warns. - MinVRAM string `json:"min_vram,omitempty" yaml:"min_vram,omitempty"` + // Variants is an optional, UNORDERED list of alternative builds of the same + // model (other backends such as MLX or vLLM, other quantizations) that the + // installer may pick instead of this entry's own payload. Authoring is + // deliberately dumb: name the models, and the selector works out which one + // this host should get. + // + // The entry itself is always the last resort, so an entry carrying variants + // stays a complete, installable entry and older LocalAI releases, which drop + // this key, install it exactly as before. + Variants []Variant `json:"variants,omitempty" yaml:"variants,omitempty"` + // MinMemory is this entry's own memory requirement (e.g. "2GiB"), in the + // same form as a variant's. It is advisory for the base entry: falling short + // only warns, because the base always installs. + MinMemory string `json:"min_memory,omitempty" yaml:"min_memory,omitempty"` } func (m *GalleryModel) GetInstalled() bool { @@ -58,12 +59,11 @@ func (m GalleryModel) ID() string { return fmt.Sprintf("%s@%s", m.Gallery.Name, m.Name) } -// HasCandidates reports whether this entry declares hardware-gated upgrades -// over its own payload. It says nothing about the entry being installable: -// an entry with candidates is a complete entry that can always be installed -// as-is. -func (m GalleryModel) HasCandidates() bool { - return len(m.Candidates) > 0 +// HasVariants reports whether this entry declares alternative builds of itself. +// It says nothing about the entry being installable: an entry with variants is +// a complete entry that can always be installed as-is. +func (m GalleryModel) HasVariants() bool { + return len(m.Variants) > 0 } func (m *GalleryModel) GetTags() []string { diff --git a/core/gallery/resolve_candidate.go b/core/gallery/resolve_candidate.go deleted file mode 100644 index c8574590ac3a..000000000000 --- a/core/gallery/resolve_candidate.go +++ /dev/null @@ -1,84 +0,0 @@ -package gallery - -import ( - "errors" - "fmt" - "strings" -) - -var ( - // ErrNoCandidateMatch is returned when no candidate satisfies the host. - ErrNoCandidateMatch = errors.New("no model variant matches this system") - // ErrPinNotFound is returned when a pinned variant is absent from the list. - ErrPinNotFound = errors.New("pinned model variant not found") -) - -// ResolveEnv describes the host properties a candidate is matched against. -// -// It is a plain struct rather than a *SystemState so the resolver stays pure -// and testable across hardware shapes without touching the machine. -type ResolveEnv struct { - // Capability is the raw value from SystemState.DetectedCapability(). - Capability string - // VRAM is total available VRAM in bytes, already capped by the operator - // VRAM budget because the cap is applied inside detection. - VRAM uint64 -} - -// ResolveCandidate returns the first candidate the host satisfies, or the -// pinned candidate when pin is non-empty. -// -// Why first-match rather than best-match: the list order is the policy. It is -// authored deliberately and lint-checked, which keeps selection explainable -// (you can read the list top to bottom) and keeps fallback expressible without -// a scoring function nobody can predict. -func ResolveCandidate(candidates []Candidate, env ResolveEnv, pin string) (Candidate, error) { - if pin != "" { - for _, c := range candidates { - if strings.EqualFold(c.Model, pin) { - return c, nil - } - } - return Candidate{}, fmt.Errorf("%w: %q is not among the %d variants of this model", ErrPinNotFound, pin, len(candidates)) - } - - reasons := make([]string, 0, len(candidates)) - for _, c := range candidates { - if c.Capability != "" && c.Capability != env.Capability { - reasons = append(reasons, fmt.Sprintf("%s requires capability %q", c.Model, c.Capability)) - continue - } - - floor, declared, err := c.EffectiveMinVRAM() - if err != nil { - return Candidate{}, err - } - // A candidate with no declared floor states no requirement and - // passes. Lint guarantees only the final last-resort candidate is - // in that position for the official gallery. - if declared && env.VRAM < floor { - reasons = append(reasons, fmt.Sprintf("%s needs %s VRAM", c.Model, humanBytes(floor))) - continue - } - - return c, nil - } - - return Candidate{}, fmt.Errorf( - "%w: capability %q with %s VRAM available; candidates: %s", - ErrNoCandidateMatch, env.Capability, humanBytes(env.VRAM), strings.Join(reasons, "; "), - ) -} - -func humanBytes(b uint64) string { - const unit = 1024 - if b < unit { - return fmt.Sprintf("%dB", b) - } - div, exp := uint64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f%ciB", float64(b)/float64(div), "KMGTPE"[exp]) -} diff --git a/core/gallery/resolve_candidate_test.go b/core/gallery/resolve_candidate_test.go deleted file mode 100644 index 4ecdc6238236..000000000000 --- a/core/gallery/resolve_candidate_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package gallery_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/mudler/LocalAI/core/gallery" -) - -var _ = Describe("Candidate.EffectiveMinVRAM", func() { - It("reports no floor when neither authored nor inferred", func() { - c := gallery.Candidate{Model: "x"} - bytes, declared, err := c.EffectiveMinVRAM() - Expect(err).ToNot(HaveOccurred()) - Expect(declared).To(BeFalse()) - Expect(bytes).To(Equal(uint64(0))) - }) - - It("uses the inferred value when nothing is authored", func() { - c := gallery.Candidate{Model: "x", InferredMinVRAM: "6GiB"} - bytes, declared, err := c.EffectiveMinVRAM() - Expect(err).ToNot(HaveOccurred()) - Expect(declared).To(BeTrue()) - Expect(bytes).To(Equal(uint64(6 * 1024 * 1024 * 1024))) - }) - - It("lets an authored value win over an inferred one", func() { - c := gallery.Candidate{Model: "x", MinVRAM: "20GiB", InferredMinVRAM: "6GiB"} - bytes, declared, err := c.EffectiveMinVRAM() - Expect(err).ToNot(HaveOccurred()) - Expect(declared).To(BeTrue()) - Expect(bytes).To(Equal(uint64(20 * 1024 * 1024 * 1024))) - }) - - It("errors on an unparseable floor rather than silently treating it as absent", func() { - c := gallery.Candidate{Model: "x", MinVRAM: "twenty gigs"} - _, _, err := c.EffectiveMinVRAM() - Expect(err).To(HaveOccurred()) - }) -}) - -var _ = Describe("ResolveCandidate", func() { - gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 } - - list := []gallery.Candidate{ - {Model: "m-mlx", Capability: "metal", MinVRAM: "16GiB"}, - {Model: "m-vllm", Capability: "nvidia", MinVRAM: "20GiB"}, - {Model: "m-q8", MinVRAM: "12GiB"}, - {Model: "m-q4", MinVRAM: "6GiB"}, - {Model: "m-q4-cpu"}, - } - - It("picks the capability-matched candidate when VRAM allows", func() { - got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") - Expect(err).ToNot(HaveOccurred()) - Expect(got.Model).To(Equal("m-vllm")) - }) - - It("skips a capability match whose VRAM floor does not fit", func() { - got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(8)}, "") - Expect(err).ToNot(HaveOccurred()) - Expect(got.Model).To(Equal("m-q4")) - }) - - It("ignores candidates whose capability does not match the host", func() { - got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "metal", VRAM: gib(24)}, "") - Expect(err).ToNot(HaveOccurred()) - Expect(got.Model).To(Equal("m-mlx")) - }) - - It("falls through to the unconstrained last resort on a tiny host", func() { - got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "default", VRAM: gib(2)}, "") - Expect(err).ToNot(HaveOccurred()) - Expect(got.Model).To(Equal("m-q4-cpu")) - }) - - It("honors a pin even when the host would have chosen otherwise", func() { - got, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "m-q8") - Expect(err).ToNot(HaveOccurred()) - Expect(got.Model).To(Equal("m-q8")) - }) - - It("fails loudly when the pin names an absent candidate", func() { - _, err := gallery.ResolveCandidate(list, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "m-gone") - Expect(err).To(MatchError(gallery.ErrPinNotFound)) - Expect(err.Error()).To(ContainSubstring("m-gone")) - }) - - It("reports the host state when nothing matches", func() { - constrained := []gallery.Candidate{{Model: "big", MinVRAM: "80GiB"}} - _, err := gallery.ResolveCandidate(constrained, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") - Expect(err).To(MatchError(gallery.ErrNoCandidateMatch)) - Expect(err.Error()).To(ContainSubstring("default")) - Expect(err.Error()).To(ContainSubstring("big")) - }) - - It("propagates an unparseable floor instead of treating it as unconstrained", func() { - bad := []gallery.Candidate{{Model: "bad", MinVRAM: "lots"}} - _, err := gallery.ResolveCandidate(bad, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("bad")) - }) - - It("rejects an empty candidate list", func() { - _, err := gallery.ResolveCandidate(nil, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") - Expect(err).To(MatchError(gallery.ErrNoCandidateMatch)) - }) -}) diff --git a/core/gallery/resolve_variant.go b/core/gallery/resolve_variant.go new file mode 100644 index 000000000000..cdfceeae6283 --- /dev/null +++ b/core/gallery/resolve_variant.go @@ -0,0 +1,162 @@ +package gallery + +import ( + "errors" + "fmt" + "sort" + "strings" +) + +var ( + // ErrNoVariantMatch is returned when nothing at all is selectable, which + // only happens for a caller that supplies no base option. + ErrNoVariantMatch = errors.New("no model variant matches this system") + // ErrPinNotFound is returned when a pinned variant is absent from the list. + ErrPinNotFound = errors.New("pinned model variant not found") +) + +// VariantOption is a variant paired with the facts a host is matched against. +// +// The install layer builds these by looking the referenced entries up in the +// live gallery. The selector never touches the catalog, which is what keeps it +// pure and testable across hardware shapes without touching the machine. +type VariantOption struct { + Variant Variant + // Backend is the engine the referenced entry resolves to (e.g. "llama-cpp", + // "mlx", "vllm"). Hardware support is derived from this name alone, which is + // precisely why a gallery author never has to describe hardware. + Backend string + // IsBase marks the declaring entry's own payload. It is exempt from every + // filter and is the answer when nothing else survives, because the entry + // must stay installable on every host and for every client. + IsBase bool +} + +// ResolveEnv describes the host a variant is selected for. +type ResolveEnv struct { + // AvailableMemory is what a model may occupy on this host: VRAM when a + // usable GPU was detected, system RAM otherwise. A model's footprint is + // roughly the same in either, so one number and one comparison cover both. + AvailableMemory uint64 + // BackendCompatible reports whether a backend can run here. The install + // layer wires SystemState.IsBackendCompatible, which already derives + // Darwin-only, NVIDIA-only, ROCm-only and SYCL-only from the backend name. + // + // A nil func treats every backend as runnable, the right default for a + // caller with no view of the hardware. + BackendCompatible func(backend string) bool +} + +func (e ResolveEnv) backendRuns(backend string) bool { + if e.BackendCompatible == nil { + return true + } + return e.BackendCompatible(backend) +} + +// VariantSelection is the outcome of a selection pass. +type VariantSelection struct { + Option VariantOption + // FellBackToBase reports that no declared variant survived and the entry's + // own payload was chosen instead. Callers log this, because a host quietly + // taking the base when upgrades were on offer is worth being able to see. + FellBackToBase bool + // Reasons explains, one line per rejected variant, why it was dropped. + Reasons []string +} + +// SelectVariant picks the option to install for a host. +// +// The algorithm, in order: +// +// 1. An explicit pin wins outright, fit or no fit. It is a deliberate operator +// override, so refusing it would defeat the point; the caller warns. +// 2. Variants whose backend cannot run here are dropped. This is the whole of +// the hardware gate, derived from the backend name. +// 3. Variants whose known memory requirement exceeds what the host has are +// dropped. A variant with an UNKNOWN requirement survives, because nothing +// proves it does not fit and refusing on a missing estimate would punish +// the entries the denormalizer has not reached yet. +// 4. The largest survivor wins. A bigger footprint is a higher quality +// quantization of the same model, so among things that fit, more is better. +// Unknown requirements rank last, so a proven fit always beats a guess. +// 5. With no survivor the base option wins. The base always installs; this +// never refuses. +func SelectVariant(options []VariantOption, env ResolveEnv, pin string) (VariantSelection, error) { + if pin != "" { + for _, o := range options { + if strings.EqualFold(o.Variant.Model, pin) { + return VariantSelection{Option: o}, nil + } + } + return VariantSelection{}, fmt.Errorf("%w: %q is not among the %d variants of this model", ErrPinNotFound, pin, len(options)) + } + + type ranked struct { + option VariantOption + memory uint64 + known bool + } + + var base *VariantOption + survivors := make([]ranked, 0, len(options)) + reasons := make([]string, 0, len(options)) + + for i := range options { + o := options[i] + if o.IsBase { + base = &options[i] + continue + } + + if !env.backendRuns(o.Backend) { + reasons = append(reasons, fmt.Sprintf("%s needs backend %q, which cannot run on this system", o.Variant.Model, o.Backend)) + continue + } + + memory, known, err := o.Variant.EffectiveMinMemory() + if err != nil { + return VariantSelection{}, err + } + if known && memory > env.AvailableMemory { + reasons = append(reasons, fmt.Sprintf("%s needs %s of memory", o.Variant.Model, humanBytes(memory))) + continue + } + + survivors = append(survivors, ranked{option: o, memory: memory, known: known}) + } + + if len(survivors) > 0 { + // Stable so that variants with identical requirements keep their + // authored order, which is the only thing order still decides. + sort.SliceStable(survivors, func(i, j int) bool { + if survivors[i].known != survivors[j].known { + return survivors[i].known + } + return survivors[i].memory > survivors[j].memory + }) + return VariantSelection{Option: survivors[0].option, Reasons: reasons}, nil + } + + if base != nil { + return VariantSelection{Option: *base, FellBackToBase: true, Reasons: reasons}, nil + } + + return VariantSelection{}, fmt.Errorf( + "%w: %s of memory available; variants: %s", + ErrNoVariantMatch, humanBytes(env.AvailableMemory), strings.Join(reasons, "; "), + ) +} + +func humanBytes(b uint64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%dB", b) + } + div, exp := uint64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f%ciB", float64(b)/float64(div), "KMGTPE"[exp]) +} diff --git a/core/gallery/resolve_variant_test.go b/core/gallery/resolve_variant_test.go new file mode 100644 index 000000000000..190fe3c1e90c --- /dev/null +++ b/core/gallery/resolve_variant_test.go @@ -0,0 +1,273 @@ +package gallery_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/gallery" +) + +var _ = Describe("Variant.EffectiveMinMemory", func() { + It("reports no requirement when neither authored nor inferred", func() { + v := gallery.Variant{Model: "x"} + size, known, err := v.EffectiveMinMemory() + Expect(err).ToNot(HaveOccurred()) + Expect(known).To(BeFalse()) + Expect(size).To(Equal(uint64(0))) + }) + + It("uses the inferred value when nothing is authored", func() { + v := gallery.Variant{Model: "x", InferredMinMemory: "6GiB"} + size, known, err := v.EffectiveMinMemory() + Expect(err).ToNot(HaveOccurred()) + Expect(known).To(BeTrue()) + Expect(size).To(Equal(uint64(6 * 1024 * 1024 * 1024))) + }) + + It("lets an authored value win over an inferred one", func() { + v := gallery.Variant{Model: "x", MinMemory: "20GiB", InferredMinMemory: "6GiB"} + size, known, err := v.EffectiveMinMemory() + Expect(err).ToNot(HaveOccurred()) + Expect(known).To(BeTrue()) + Expect(size).To(Equal(uint64(20 * 1024 * 1024 * 1024))) + }) + + It("errors on an unparseable figure rather than silently treating it as absent", func() { + v := gallery.Variant{Model: "x", MinMemory: "twenty gigs"} + _, _, err := v.EffectiveMinMemory() + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("SelectVariant", func() { + gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 } + + option := func(model, backend, minMemory string) gallery.VariantOption { + return gallery.VariantOption{ + Variant: gallery.Variant{Model: model, MinMemory: minMemory}, + Backend: backend, + } + } + + base := func(model, minMemory string) gallery.VariantOption { + o := option(model, "llama-cpp", minMemory) + o.IsBase = true + return o + } + + // linuxNvidia mirrors what SystemState.IsBackendCompatible does on a Linux + // box with an NVIDIA card: Darwin-only engines are out, everything else runs. + linuxNvidia := func(backend string) bool { + return backend != "mlx" && backend != "mlx-vlm" + } + + Describe("hardware filtering", func() { + It("never selects a variant whose backend cannot run on this host", func() { + // The MLX build is both the largest and the only thing that would + // otherwise win, so nothing but the backend gate can reject it. + options := []gallery.VariantOption{ + option("m-mlx-8bit", "mlx", "24GiB"), + option("m-gguf-q8", "llama-cpp", "12GiB"), + base("m-gguf-q4", "6GiB"), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ + AvailableMemory: gib(80), + BackendCompatible: linuxNvidia, + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-gguf-q8")) + }) + + It("selects the same variant on a host whose backend gate does admit it", func() { + // The mirror image of the spec above, so the rejection is proven to + // come from the host and not from something intrinsic to the entry. + options := []gallery.VariantOption{ + option("m-mlx-8bit", "mlx", "24GiB"), + option("m-gguf-q8", "llama-cpp", "12GiB"), + base("m-gguf-q4", "6GiB"), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ + AvailableMemory: gib(80), + BackendCompatible: func(string) bool { return true }, + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-mlx-8bit")) + }) + + It("treats every backend as runnable when the host cannot be inspected", func() { + options := []gallery.VariantOption{option("m-mlx-8bit", "mlx", "24GiB"), base("m-gguf-q4", "6GiB")} + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(80)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-mlx-8bit")) + }) + }) + + Describe("ranking", func() { + It("picks the largest variant that fits, not the first authored", func() { + // Authored smallest-first, so first-match would take m-q4 and any + // ranking that ignores size would too. + options := []gallery.VariantOption{ + option("m-q4", "llama-cpp", "6GiB"), + option("m-q8", "llama-cpp", "12GiB"), + option("m-f16", "llama-cpp", "24GiB"), + base("m-base", "2GiB"), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-q8")) + Expect(selection.FellBackToBase).To(BeFalse()) + }) + + It("picks the largest variant regardless of authored order", func() { + // Same set, authored largest-first. Order must make no difference at + // all, which is the entire point of dropping ordered first-match. + options := []gallery.VariantOption{ + option("m-f16", "llama-cpp", "24GiB"), + option("m-q8", "llama-cpp", "12GiB"), + option("m-q4", "llama-cpp", "6GiB"), + base("m-base", "2GiB"), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-q8")) + }) + + It("prefers a known fit over a variant of unknown size", func() { + // An unknown requirement is a guess. It survives the filter, because + // nothing proves it does not fit, but it must never displace a + // variant that is known to fit. + options := []gallery.VariantOption{ + option("m-unknown", "llama-cpp", ""), + option("m-q8", "llama-cpp", "12GiB"), + base("m-base", "2GiB"), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-q8")) + }) + + It("keeps a variant of unknown size rather than dropping it", func() { + options := []gallery.VariantOption{ + option("m-unknown", "llama-cpp", ""), + base("m-base", "2GiB"), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(4)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-unknown")) + Expect(selection.FellBackToBase).To(BeFalse()) + }) + + It("admits a variant needing exactly the memory available", func() { + options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", "2GiB")} + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(12)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-q8")) + }) + }) + + Describe("falling back to the base", func() { + It("selects the base when nothing else fits", func() { + options := []gallery.VariantOption{ + option("m-q8", "llama-cpp", "12GiB"), + option("m-f16", "llama-cpp", "24GiB"), + base("m-base", "2GiB"), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(4)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-base")) + Expect(selection.Option.IsBase).To(BeTrue()) + Expect(selection.FellBackToBase).To(BeTrue()) + Expect(selection.Reasons).To(HaveLen(2)) + }) + + It("selects the base even when the base's own requirement is unmet", func() { + // There is nothing below the base, so refusing here would make an + // entry every older client installs fine uninstallable on newer ones. + options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", "2GiB")} + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: 0}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-base")) + }) + + It("explains why each variant was rejected", func() { + options := []gallery.VariantOption{ + option("m-mlx", "mlx", "8GiB"), + option("m-f16", "llama-cpp", "24GiB"), + base("m-base", "2GiB"), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ + AvailableMemory: gib(4), + BackendCompatible: linuxNvidia, + }, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Reasons).To(ContainElement(ContainSubstring("cannot run on this system"))) + Expect(selection.Reasons).To(ContainElement(ContainSubstring("24.0GiB"))) + }) + + It("errors when the caller supplies no base at all", func() { + options := []gallery.VariantOption{option("m-f16", "llama-cpp", "24GiB")} + + _, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(4)}, "") + Expect(err).To(MatchError(gallery.ErrNoVariantMatch)) + }) + }) + + Describe("explicit selection", func() { + It("honors a pin the hardware would never have chosen", func() { + options := []gallery.VariantOption{ + option("m-mlx", "mlx", "64GiB"), + base("m-base", "2GiB"), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ + AvailableMemory: gib(4), + BackendCompatible: linuxNvidia, + }, "m-mlx") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-mlx")) + }) + + It("honors a pin naming the base, declining an upgrade that fits", func() { + options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", "2GiB")} + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "m-base") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-base")) + }) + + It("matches a pin case-insensitively", func() { + options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", "2GiB")} + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "M-F16") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-f16")) + }) + + It("fails loudly when the pin names nothing in the list", func() { + options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", "2GiB")} + + _, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "m-gone") + Expect(err).To(MatchError(gallery.ErrPinNotFound)) + Expect(err.Error()).To(ContainSubstring("m-gone")) + }) + }) + + It("propagates an unparseable figure instead of treating it as unconstrained", func() { + options := []gallery.VariantOption{option("bad", "llama-cpp", "lots"), base("m-base", "2GiB")} + + _, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(8)}, "") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("bad")) + }) +}) diff --git a/core/gallery/variant.go b/core/gallery/variant.go new file mode 100644 index 000000000000..0821f440a4c4 --- /dev/null +++ b/core/gallery/variant.go @@ -0,0 +1,56 @@ +package gallery + +import ( + "fmt" + + "github.com/mudler/LocalAI/pkg/vram" +) + +// Variant is one option in a gallery entry's variant list. It references an +// existing gallery entry by name, and that is all an author has to write. +// +// Authored order carries no meaning. Selection filters out what this host +// cannot run and then ranks what is left, so hardware knowledge lives in the +// selector rather than being pushed onto whoever edits the gallery. +type Variant struct { + // Model is the name of a gallery entry that declares no variants of its own. + Model string `json:"model" yaml:"model"` + // MinMemory is the authored memory requirement (e.g. "20GiB"). It exists to + // override an inferred estimate that is known to be wrong; most variants + // should not need it. + // + // One number covers both VRAM and system RAM. A model's footprint is + // roughly the same wherever it lives, and the selector compares this against + // whichever of the two this host will actually use, so splitting it in two + // would only invite the pair to disagree. + MinMemory string `json:"min_memory,omitempty" yaml:"min_memory,omitempty"` + + // The fields below are denormalized by the nightly job for display and + // lint. They are never authored by hand and never affect what gets + // installed, because installation reads the referenced entry live. + Backend string `json:"backend,omitempty" yaml:"backend,omitempty"` + Quantization string `json:"quantization,omitempty" yaml:"quantization,omitempty"` + InferredMinMemory string `json:"inferred_min_memory,omitempty" yaml:"inferred_min_memory,omitempty"` +} + +// EffectiveMinMemory returns this variant's memory requirement in bytes and +// whether one is known at all. An authored MinMemory wins over an inferred one, +// because a human who measured a real load knows more than a pre-download +// estimate does. +// +// An unknown requirement is not a zero requirement. Callers must not read the +// returned 0 as "fits anywhere"; it means nothing is known either way. +func (v Variant) EffectiveMinMemory() (uint64, bool, error) { + raw := v.MinMemory + if raw == "" { + raw = v.InferredMinMemory + } + if raw == "" { + return 0, false, nil + } + size, err := vram.ParseSizeString(raw) + if err != nil { + return 0, false, fmt.Errorf("variant %q has an unparseable min_memory %q: %w", v.Model, raw, err) + } + return size, true, nil +} diff --git a/core/gallery/candidates_install_test.go b/core/gallery/variants_install_test.go similarity index 61% rename from core/gallery/candidates_install_test.go rename to core/gallery/variants_install_test.go index 108f26e6605b..1de4e5ace432 100644 --- a/core/gallery/candidates_install_test.go +++ b/core/gallery/variants_install_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "runtime" "dario.cat/mergo" . "github.com/onsi/ginkgo/v2" @@ -15,50 +16,51 @@ import ( "github.com/mudler/LocalAI/pkg/system" ) -var _ = Describe("GalleryModel candidate declarations", func() { - It("declares no candidates when the key is absent", func() { +var _ = Describe("GalleryModel variant declarations", func() { + It("declares no variants when the key is absent", func() { m := gallery.GalleryModel{} m.Name = "plain" - Expect(m.HasCandidates()).To(BeFalse()) + Expect(m.HasVariants()).To(BeFalse()) }) - It("declares candidates when the key is present", func() { - m := gallery.GalleryModel{Candidates: []gallery.Candidate{{Model: "x"}}} - Expect(m.HasCandidates()).To(BeTrue()) + It("declares variants when the key is present", func() { + m := gallery.GalleryModel{Variants: []gallery.Variant{{Model: "x"}}} + Expect(m.HasVariants()).To(BeTrue()) }) - It("parses an entry's own selection fields and its candidate list in order", func() { + It("parses an entry's own memory figure and its variant list", func() { var m gallery.GalleryModel err := yaml.Unmarshal([]byte(` -name: qwen3-8b-gguf-q4 -url: "github:example/repo/qwen3-8b-gguf-q4.yaml@master" -min_vram: 6GiB -capability: default -candidates: - - model: qwen3-8b-vllm-awq - capability: nvidia - min_vram: 20GiB - - model: qwen3-8b-gguf-q8 - min_vram: 10GiB +name: qwen3.6-27b +url: "github:example/repo/qwen3.6-27b.yaml@master" +min_memory: 4GiB +variants: + - model: qwen3.6-27b-mlx-8bit + - model: qwen3.6-27b-gguf-q8 + min_memory: 28GiB `), &m) Expect(err).ToNot(HaveOccurred()) - Expect(m.Name).To(Equal("qwen3-8b-gguf-q4")) - Expect(m.URL).To(Equal("github:example/repo/qwen3-8b-gguf-q4.yaml@master")) - Expect(m.MinVRAM).To(Equal("6GiB")) - Expect(m.Capability).To(Equal("default")) - Expect(m.HasCandidates()).To(BeTrue()) - Expect(m.Candidates).To(HaveLen(2)) - Expect(m.Candidates[0].Model).To(Equal("qwen3-8b-vllm-awq")) - Expect(m.Candidates[0].Capability).To(Equal("nvidia")) - Expect(m.Candidates[0].MinVRAM).To(Equal("20GiB")) - Expect(m.Candidates[1].Model).To(Equal("qwen3-8b-gguf-q8")) - Expect(m.Candidates[1].Capability).To(BeEmpty()) + Expect(m.Name).To(Equal("qwen3.6-27b")) + Expect(m.URL).To(Equal("github:example/repo/qwen3.6-27b.yaml@master")) + Expect(m.MinMemory).To(Equal("4GiB")) + Expect(m.HasVariants()).To(BeTrue()) + Expect(m.Variants).To(HaveLen(2)) + // The first variant is nothing but a name, which is the shape authoring + // is meant to reach for. + Expect(m.Variants[0].Model).To(Equal("qwen3.6-27b-mlx-8bit")) + Expect(m.Variants[0].MinMemory).To(BeEmpty()) + Expect(m.Variants[1].Model).To(Equal("qwen3.6-27b-gguf-q8")) + Expect(m.Variants[1].MinMemory).To(Equal("28GiB")) }) }) var _ = Describe("ResolveVariant", func() { gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 } + // runsEverything keeps these specs about resolution rather than about the + // hardware of whatever machine runs them. + runsEverything := func(string) bool { return true } + newModel := func(name, url, description, icon string) *gallery.GalleryModel { m := &gallery.GalleryModel{} m.Name = name @@ -73,54 +75,67 @@ var _ = Describe("ResolveVariant", func() { BeforeEach(func() { upgrade := newModel("qwen3-8b-vllm-awq", "file://vllm.yaml", "AWQ variant", "vllm.png") - // The base is an ordinary, complete entry that happens to declare an - // upgrade over itself, which is the whole point of the design. + upgrade.Backend = "vllm" + // The base is an ordinary, complete entry that happens to list an + // alternative build of itself, which is the whole point of the design. base = newModel("qwen3-8b-gguf-q4", "file://gguf.yaml", "Qwen3 8B Q4", "qwen.png") + base.Backend = "llama-cpp" base.Tags = []string{"llm"} - base.MinVRAM = "6GiB" - base.Candidates = []gallery.Candidate{ - {Model: "qwen3-8b-vllm-awq", Capability: "nvidia", MinVRAM: "20GiB"}, - } + base.MinMemory = "6GiB" + base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq", MinMemory: "20GiB"}} models = []*gallery.GalleryModel{upgrade, base} }) - It("installs a matching candidate's payload under the entry's name", func() { - resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + env := func(memory uint64) gallery.ResolveEnv { + return gallery.ResolveEnv{AvailableMemory: memory, BackendCompatible: runsEverything} + } + + It("installs a fitting variant's payload under the entry's name", func() { + resolved, variant, err := gallery.ResolveVariant(models, base, env(gib(24)), "") Expect(err).ToNot(HaveOccurred()) - Expect(candidate.Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(variant.Model).To(Equal("qwen3-8b-vllm-awq")) Expect(resolved.Name).To(Equal("qwen3-8b-gguf-q4")) Expect(resolved.URL).To(Equal("file://vllm.yaml")) }) - It("falls back to the entry's own payload when no candidate fits", func() { - resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + It("carries the referenced entry's backend into the compatibility check", func() { + // The variant itself says nothing about hardware, so the backend can + // only come from the entry it names. Rejecting exactly that backend must + // therefore be enough to rule the variant out. + resolved, variant, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{ + AvailableMemory: gib(64), + BackendCompatible: func(backend string) bool { return backend != "vllm" }, + }, "") Expect(err).ToNot(HaveOccurred()) - Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4")) Expect(resolved.URL).To(Equal("file://gguf.yaml")) }) - It("installs the entry even when the host misses the entry's own floor", func() { - // There is nothing below the base, so refusing here would make an entry - // that every older client installs fine uninstallable on new ones. - resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(1)}, "") + It("falls back to the entry's own payload when no variant fits", func() { + resolved, variant, err := gallery.ResolveVariant(models, base, env(gib(8)), "") Expect(err).ToNot(HaveOccurred()) - Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) + }) + + It("installs the entry even when the host misses the entry's own requirement", func() { + resolved, variant, err := gallery.ResolveVariant(models, base, env(gib(1)), "") + Expect(err).ToNot(HaveOccurred()) + Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4")) Expect(resolved.URL).To(Equal("file://gguf.yaml")) }) It("strips the selection fields from the resolved entry", func() { // A resolved entry is a concrete install target. Leaving the fields on - // it would let a second resolution pass fire on an already-resolved - // entry. - resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + // it would let a second selection pass fire on an already-resolved entry. + resolved, _, err := gallery.ResolveVariant(models, base, env(gib(8)), "") Expect(err).ToNot(HaveOccurred()) - Expect(resolved.HasCandidates()).To(BeFalse()) - Expect(resolved.MinVRAM).To(BeEmpty()) - Expect(resolved.Capability).To(BeEmpty()) + Expect(resolved.HasVariants()).To(BeFalse()) + Expect(resolved.MinMemory).To(BeEmpty()) }) - It("presents the entry's metadata, not the candidate's", func() { - resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + It("presents the entry's metadata, not the variant's", func() { + resolved, _, err := gallery.ResolveVariant(models, base, env(gib(24)), "") Expect(err).ToNot(HaveOccurred()) Expect(resolved.Description).To(Equal("Qwen3 8B Q4")) Expect(resolved.Icon).To(Equal("qwen.png")) @@ -128,20 +143,20 @@ var _ = Describe("ResolveVariant", func() { }) It("honors a pin naming the entry itself", func() { - // The entry is the last element of its own candidate list, so its own - // name has to be a usable pin: it is how an operator declines an - // upgrade their hardware would otherwise take. - resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "qwen3-8b-gguf-q4") + // The entry is the last resort of its own variant list, so its own name + // has to be a usable pin: it is how an operator declines an upgrade + // their hardware would otherwise take. + resolved, variant, err := gallery.ResolveVariant(models, base, env(gib(24)), "qwen3-8b-gguf-q4") Expect(err).ToNot(HaveOccurred()) - Expect(candidate.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4")) Expect(resolved.Name).To(Equal("qwen3-8b-gguf-q4")) Expect(resolved.URL).To(Equal("file://gguf.yaml")) }) It("honors a pin the hardware does not satisfy", func() { - resolved, candidate, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(2)}, "qwen3-8b-vllm-awq") + resolved, variant, err := gallery.ResolveVariant(models, base, env(gib(2)), "qwen3-8b-vllm-awq") Expect(err).ToNot(HaveOccurred()) - Expect(candidate.Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(variant.Model).To(Equal("qwen3-8b-vllm-awq")) Expect(resolved.URL).To(Equal("file://vllm.yaml")) }) @@ -149,7 +164,7 @@ var _ = Describe("ResolveVariant", func() { models[0].Overrides = map[string]any{"f16": true} models[0].AdditionalFiles = []gallery.File{{Filename: "a.bin"}} - resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + resolved, _, err := gallery.ResolveVariant(models, base, env(gib(24)), "") Expect(err).ToNot(HaveOccurred()) // The install path merges the caller's request overrides into this map in @@ -172,7 +187,7 @@ var _ = Describe("ResolveVariant", func() { // holds, which is the case most likely to alias it. base.Overrides = map[string]any{"parameters": map[string]any{"model": "q4.gguf"}} - resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + resolved, _, err := gallery.ResolveVariant(models, base, env(gib(8)), "") Expect(err).ToNot(HaveOccurred()) resolved.Overrides["parameters"].(map[string]any)["model"] = "callers-choice.gguf" @@ -185,7 +200,7 @@ var _ = Describe("ResolveVariant", func() { "stopwords": []any{""}, } - resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + resolved, _, err := gallery.ResolveVariant(models, base, env(gib(24)), "") Expect(err).ToNot(HaveOccurred()) // Cloning only the top level would leave this inner map shared with the @@ -201,7 +216,7 @@ var _ = Describe("ResolveVariant", func() { It("does not write the caller's overrides back into the gallery entry", func() { models[0].Overrides = map[string]any{"parameters": map[string]any{"model": "real-variant.gguf"}} - resolved, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "") + resolved, _, err := gallery.ResolveVariant(models, base, env(gib(24)), "") Expect(err).ToNot(HaveOccurred()) // This is exactly what the install path does with the caller's request @@ -216,30 +231,30 @@ var _ = Describe("ResolveVariant", func() { Expect(models[0].Overrides["parameters"]).To(HaveKeyWithValue("model", "real-variant.gguf")) }) - It("errors when a candidate references a missing entry", func() { - base.Candidates = []gallery.Candidate{{Model: "does-not-exist"}} - _, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + It("errors when a variant references a missing entry", func() { + base.Variants = []gallery.Variant{{Model: "does-not-exist"}} + _, _, err := gallery.ResolveVariant(models, base, env(gib(8)), "") Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("does-not-exist")) }) - It("refuses a candidate that declares candidates of its own", func() { + It("refuses a variant that declares variants of its own", func() { nested := newModel("nested", "file://nested.yaml", "", "") - nested.Candidates = []gallery.Candidate{{Model: "qwen3-8b-vllm-awq"}} + nested.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq"}} models = append(models, nested) - base.Candidates = []gallery.Candidate{{Model: "nested"}} - _, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "default", VRAM: gib(8)}, "") + base.Variants = []gallery.Variant{{Model: "nested"}} + _, _, err := gallery.ResolveVariant(models, base, env(gib(8)), "") Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("nested")) }) It("surfaces a bad pin", func() { - _, _, err := gallery.ResolveVariant(models, base, gallery.ResolveEnv{Capability: "nvidia", VRAM: gib(24)}, "nope") + _, _, err := gallery.ResolveVariant(models, base, env(gib(24)), "nope") Expect(err).To(MatchError(gallery.ErrPinNotFound)) }) }) -var _ = Describe("InstallModelFromGallery with candidate entries", func() { +var _ = Describe("InstallModelFromGallery with variant entries", func() { var tempdir string var galleries []config.Gallery var systemState *system.SystemState @@ -286,12 +301,12 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() { return m } - // withCandidates attaches upgrades to an otherwise ordinary entry. The - // floors are absolute rather than relative to the host: "0GiB" always - // matches and "10000GiB" never does, so these specs assert on resolution - // rather than on whatever VRAM the machine running them happens to have. - withCandidates := func(m gallery.GalleryModel, candidates ...gallery.Candidate) gallery.GalleryModel { - m.Candidates = candidates + // withVariants attaches alternative builds to an otherwise ordinary entry. + // The figures are absolute rather than relative to the host: "0GiB" always + // fits and "10000GiB" never does, so these specs assert on selection rather + // than on whatever memory the machine running them happens to have. + withVariants := func(m gallery.GalleryModel, variants ...gallery.Variant) gallery.GalleryModel { + m.Variants = variants return m } @@ -311,7 +326,7 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() { BeforeEach(func() { var err error - tempdir, err = os.MkdirTemp("", "candidate-install") + tempdir, err = os.MkdirTemp("", "variant-install") Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { Expect(os.RemoveAll(tempdir)).To(Succeed()) }) @@ -319,42 +334,73 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() { Expect(err).ToNot(HaveOccurred()) }) - It("installs the entry's own payload when no candidate fits the host", func() { + It("installs the entry's own payload when no variant fits the host", func() { newGallery( - withCandidates(entry("qwen3-8b-q4", "base-backend"), - gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "10000GiB"}), + withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "10000GiB"}), entry("qwen3-8b-q8", "upgrade-backend"), ) - // No machine clears a 10000GiB floor, so this asserts the base is the - // last resort and that missing every candidate is not an error. + // No machine clears 10000GiB, so this asserts the base is the last + // resort and that missing every variant is not an error. Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend")) }) - It("installs a fitting candidate's payload under the entry's own name", func() { + It("installs a fitting variant's payload under the entry's own name", func() { newGallery( - withCandidates(entry("qwen3-8b-q4", "base-backend"), - gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}), + withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}), entry("qwen3-8b-q8", "upgrade-backend"), ) - // A 0GiB floor is met by every machine, so this asserts the upgrade wins - // over the base and lands under the base's name rather than its own. Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) Expect(installedBackend("qwen3-8b-q4")).To(Equal("upgrade-backend")) _, err := os.Stat(filepath.Join(tempdir, "qwen3-8b-q8.yaml")) - Expect(os.IsNotExist(err)).To(BeTrue(), "the upgrade must not be installed under its own name") + Expect(os.IsNotExist(err)).To(BeTrue(), "the variant must not be installed under its own name") + }) + + It("installs the largest fitting variant, not the first authored", func() { + // Authored smallest-first with all three within reach, so first-match + // would take the small one. + newGallery( + withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-small", MinMemory: "0GiB"}, + gallery.Variant{Model: "qwen3-8b-large", MinMemory: "1KiB"}, + ), + entry("qwen3-8b-small", "small-backend"), + entry("qwen3-8b-large", "large-backend"), + ) + + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("large-backend")) + }) + + It("never installs a variant whose backend cannot run on this host", func() { + if runtime.GOOS == "darwin" { + Skip("mlx is a compatible backend on darwin, so it proves nothing here") + } + // The mlx entry is the only variant and fits trivially, so the real + // SystemState.IsBackendCompatible rejecting mlx on a non-darwin host is + // the only thing that can send this to the base. + newGallery( + withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-mlx", MinMemory: "0GiB"}), + entry("qwen3-8b-mlx", "mlx"), + ) + + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend")) }) It("round-trips the resolution record to disk under the entry's name", func() { - // The upgrade is described by url so its payload carries its own name. + // The variant is described by url so its payload carries its own name. // Without the entry-name overlay the record persists as "qwen3-8b-q8", // and the stable name the entry exists to provide is lost the moment // anything reads the record back. newGallery( - withCandidates(urlEntry("qwen3-8b-q4", "base-backend"), - gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}), + withVariants(urlEntry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}), urlEntry("qwen3-8b-q8", "upgrade-backend"), ) @@ -372,8 +418,8 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() { It("records a pin and honors it on a plain reinstall", func() { newGallery( - withCandidates(entry("qwen3-8b-q4", "base-backend"), - gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}), + withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}), entry("qwen3-8b-q8", "upgrade-backend"), ) @@ -385,7 +431,7 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() { Expect(record.ResolvedVariant).To(Equal("qwen3-8b-q4")) Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend")) - // No WithVariant this time: hardware resolution would take the upgrade, + // No WithVariant this time: hardware selection would take the upgrade, // so only the recalled pin can keep this on the base payload. Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend")) @@ -397,8 +443,8 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() { It("honors a pin recorded under a custom install name", func() { newGallery( - withCandidates(entry("qwen3-8b-q4", "base-backend"), - gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}), + withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}), entry("qwen3-8b-q8", "upgrade-backend"), ) @@ -420,8 +466,8 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() { }) It("writes each declared url only once into the persisted gallery file", func() { - base := withCandidates(entry("qwen3-8b-q4", "base-backend"), - gallery.Candidate{Model: "qwen3-8b-q8", MinVRAM: "0GiB"}) + base := withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}) base.URLs = []string{"https://example.invalid/qwen3"} newGallery(base, entry("qwen3-8b-q8", "upgrade-backend")) @@ -434,12 +480,12 @@ var _ = Describe("InstallModelFromGallery with candidate entries", func() { }) var _ = Describe("legacy client compatibility", func() { - It("keeps every entry that declares candidates installable by clients that ignore them", func() { + It("keeps every entry that declares variants installable by clients that ignore them", func() { data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) Expect(err).ToNot(HaveOccurred()) // Parse exactly as an older LocalAI release would: non-strictly, with - // no knowledge of the candidates key. Such a client installs whatever + // no knowledge of the variants key. Such a client installs whatever // payload the entry carries directly, so the entry must carry one. var legacy []struct { Name string `yaml:"name"` @@ -458,23 +504,23 @@ var _ = Describe("legacy client compatibility", func() { legacyByName[e.Name] = i } - withCandidates := 0 + withVariants := 0 for _, e := range current { - if !e.HasCandidates() { + if !e.HasVariants() { continue } - withCandidates++ + withVariants++ i, ok := legacyByName[e.Name] Expect(ok).To(BeTrue(), "entry %q vanished under a legacy parse", e.Name) old := legacy[i] - // An entry whose payload lived only in its candidates would install - // to nothing on every released LocalAI, which is precisely what - // making the entry itself the base candidate exists to prevent. + // An entry whose payload lived only in its variants would install to + // nothing on every released LocalAI, which is precisely what making + // the entry itself the last resort exists to prevent. Expect(old.URL != "" || len(old.ConfigFile) > 0).To(BeTrue(), "entry %q carries no payload of its own, so older clients would install nothing", e.Name) } - Expect(withCandidates).To(BeNumerically(">", 0), - "expected at least one entry declaring candidates in the gallery index") + Expect(withVariants).To(BeNumerically(">", 0), + "expected at least one entry declaring variants in the gallery index") }) }) diff --git a/core/gallery/variants_lint_test.go b/core/gallery/variants_lint_test.go new file mode 100644 index 000000000000..4d21d17419a3 --- /dev/null +++ b/core/gallery/variants_lint_test.go @@ -0,0 +1,281 @@ +package gallery_test + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/mudler/LocalAI/core/gallery" +) + +// variantViolation is one invariant breach found by a lint helper. Helpers +// return every breach they find instead of stopping at the first, so a single +// run names them all rather than forcing a fix-one-rerun-repeat cycle. +type variantViolation struct { + Entry string + Variant string + Detail string +} + +func (v variantViolation) String() string { + if v.Variant == "" { + return fmt.Sprintf("%s: %s", v.Entry, v.Detail) + } + return fmt.Sprintf("%s -> variant %q: %s", v.Entry, v.Variant, v.Detail) +} + +func formatViolations(violations []variantViolation) string { + lines := make([]string, 0, len(violations)) + for _, v := range violations { + lines = append(lines, " "+v.String()) + } + return "\n" + strings.Join(lines, "\n") +} + +func indexEntriesByName(entries []gallery.GalleryModel) map[string]gallery.GalleryModel { + byName := make(map[string]gallery.GalleryModel, len(entries)) + for _, e := range entries { + byName[e.Name] = e + } + return byName +} + +// checkVariantReferences verifies every variant names an existing entry that +// declares no variants of its own. Selection is a single pass, so a nested +// reference would silently ignore the inner list. +// +// This is the one structural rule left. The rules about authored order, the +// hardware vocabulary and floor relationships stopped describing real hazards +// once selection became a filter plus a ranking: an author writes names, and +// there is no longer anything about hardware they can get wrong. +func checkVariantReferences(entries []gallery.GalleryModel) []variantViolation { + byName := indexEntriesByName(entries) + var violations []variantViolation + for _, e := range entries { + if !e.HasVariants() { + continue + } + for _, v := range e.Variants { + target, ok := byName[v.Model] + if !ok { + violations = append(violations, variantViolation{Entry: e.Name, Variant: v.Model, Detail: "references unknown model"}) + continue + } + if target.HasVariants() { + violations = append(violations, variantViolation{Entry: e.Name, Variant: v.Model, Detail: "references an entry that declares variants of its own; nesting is not allowed"}) + } + } + } + return violations +} + +// checkVariantMemory verifies every declared memory figure parses, on the +// variants and on the entry itself. +// +// Absence is fine and deliberately not flagged: an unknown requirement is a +// legitimate state that selection handles by ranking the variant last. An +// UNPARSEABLE one is different, because it makes selection fail outright for +// the whole entry rather than degrade. +func checkVariantMemory(entries []gallery.GalleryModel) []variantViolation { + var violations []variantViolation + for _, e := range entries { + if !e.HasVariants() { + continue + } + base := gallery.Variant{Model: e.Name, MinMemory: e.MinMemory} + if _, _, err := base.EffectiveMinMemory(); err != nil { + violations = append(violations, variantViolation{Entry: e.Name, Detail: "has a bad min_memory: " + err.Error()}) + } + for _, v := range e.Variants { + if _, _, err := v.EffectiveMinMemory(); err != nil { + violations = append(violations, variantViolation{Entry: e.Name, Variant: v.Model, Detail: "has a bad min_memory: " + err.Error()}) + } + } + } + return violations +} + +// loadGalleryIndex parses gallery/index.yaml once for the whole suite. The +// index carries well over a thousand entries, so re-parsing it per spec is +// pure overhead. +var loadGalleryIndex = sync.OnceValues(func() ([]gallery.GalleryModel, error) { + data, err := os.ReadFile(filepath.Join("..", "..", "gallery", "index.yaml")) + if err != nil { + return nil, err + } + var entries []gallery.GalleryModel + if err := yaml.Unmarshal(data, &entries); err != nil { + return nil, err + } + return entries, nil +}) + +func plainEntry(name, url string) gallery.GalleryModel { + e := gallery.GalleryModel{} + e.Name = name + e.URL = url + return e +} + +func entryWithVariants(name, url, minMemory string, variants ...gallery.Variant) gallery.GalleryModel { + e := gallery.GalleryModel{Variants: variants, MinMemory: minMemory} + e.Name = name + e.URL = url + return e +} + +// variantFixture builds an entry with variants plus the entries it references. +// Synthetic fixtures keep the invariant logic covered however few entries in +// gallery/index.yaml declare variants; without them every index-driven spec +// below is a no-op that passes while checking nothing. +func variantFixture(base gallery.GalleryModel, referenced ...gallery.GalleryModel) []gallery.GalleryModel { + return append([]gallery.GalleryModel{base}, referenced...) +} + +var _ = Describe("gallery variant lint helpers", func() { + // An entry declares variants solely by carrying the key. GalleryBackend.IsMeta() + // has deliberately different semantics (it requires an EMPTY uri), so a + // well-meaning alignment of the two would make every helper skip every + // entry and pass silently. Assert the distinction directly. + It("treats an entry with variants as such even though it has a url", func() { + Expect(entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "big"}).HasVariants()).To(BeTrue()) + Expect(plainEntry("big", "u://big").HasVariants()).To(BeFalse()) + }) + + It("passes every invariant on a valid entry", func() { + entries := variantFixture( + entryWithVariants("base", "u://base", "4GiB", + gallery.Variant{Model: "big", MinMemory: "24GiB"}, + gallery.Variant{Model: "mid", MinMemory: "12GiB"}, + gallery.Variant{Model: "metal-big", MinMemory: "32GiB"}, + gallery.Variant{Model: "small"}, + ), + plainEntry("big", "u://big"), + plainEntry("mid", "u://mid"), + plainEntry("metal-big", "u://metal-big"), + plainEntry("small", "u://small"), + ) + + Expect(checkVariantReferences(entries)).To(BeEmpty()) + Expect(checkVariantMemory(entries)).To(BeEmpty()) + }) + + It("passes every invariant on an entry that declares no memory at all", func() { + // Authoring is meant to be nothing but a list of names, so an entry and + // its variants carrying no figures must be entirely valid. + entries := variantFixture( + entryWithVariants("base", "u://base", "", gallery.Variant{Model: "big"}), + plainEntry("big", "u://big"), + ) + + Expect(checkVariantReferences(entries)).To(BeEmpty()) + Expect(checkVariantMemory(entries)).To(BeEmpty()) + }) + + Describe("checkVariantReferences", func() { + It("flags a variant naming an entry that does not exist", func() { + entries := variantFixture( + entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "ghost"}), + plainEntry("a", "u://a"), + ) + + violations := checkVariantReferences(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Variant).To(Equal("ghost")) + Expect(violations[0].Detail).To(ContainSubstring("unknown model")) + }) + + It("flags a variant naming an entry that declares variants itself", func() { + entries := variantFixture( + entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "nested"}), + entryWithVariants("nested", "u://nested", "4GiB", gallery.Variant{Model: "a"}), + plainEntry("a", "u://a"), + ) + + violations := checkVariantReferences(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Variant).To(Equal("nested")) + Expect(violations[0].Detail).To(ContainSubstring("nesting is not allowed")) + }) + + It("reports every breach in one pass rather than stopping at the first", func() { + entries := variantFixture( + entryWithVariants("base", "u://base", "2GiB", + gallery.Variant{Model: "ghost"}, + gallery.Variant{Model: "phantom"}, + ), + ) + + Expect(checkVariantReferences(entries)).To(HaveLen(2)) + }) + }) + + Describe("checkVariantMemory", func() { + It("flags a variant whose memory figure cannot be parsed", func() { + entries := variantFixture( + entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "a", MinMemory: "lots"}), + plainEntry("a", "u://a"), + ) + + violations := checkVariantMemory(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Variant).To(Equal("a")) + Expect(violations[0].Detail).To(ContainSubstring("bad min_memory")) + }) + + It("flags an entry whose own memory figure cannot be parsed", func() { + entries := variantFixture( + entryWithVariants("base", "u://base", "eight gigs", gallery.Variant{Model: "a"}), + plainEntry("a", "u://a"), + ) + + violations := checkVariantMemory(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Entry).To(Equal("base")) + Expect(violations[0].Variant).To(BeEmpty()) + Expect(violations[0].Detail).To(ContainSubstring("bad min_memory")) + }) + + It("flags an unparseable denormalized figure just as an authored one", func() { + // The nightly job writes inferred_min_memory, so a bug there breaks + // selection exactly as a bad hand-authored value does. + entries := variantFixture( + entryWithVariants("base", "u://base", "", gallery.Variant{Model: "a", InferredMinMemory: "8 gigabytes"}), + plainEntry("a", "u://a"), + ) + + violations := checkVariantMemory(entries) + Expect(violations).To(HaveLen(1)) + Expect(violations[0].Variant).To(Equal("a")) + }) + }) +}) + +var _ = Describe("gallery/index.yaml variant invariants", Ordered, func() { + var entries []gallery.GalleryModel + + BeforeAll(func() { + var err error + entries, err = loadGalleryIndex() + Expect(err).ToNot(HaveOccurred()) + // A truncated or emptied index unmarshals cleanly and would make every + // spec below vacuously pass. + Expect(entries).ToNot(BeEmpty()) + }) + + It("references only existing entries that declare no variants themselves", func() { + v := checkVariantReferences(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) + + It("declares only parseable memory figures", func() { + v := checkVariantMemory(entries) + Expect(v).To(BeEmpty(), formatViolations(v)) + }) +}) diff --git a/core/schema/gallery-model.schema.json b/core/schema/gallery-model.schema.json index d72004fbd89e..998da2ec307d 100644 --- a/core/schema/gallery-model.schema.json +++ b/core/schema/gallery-model.schema.json @@ -56,17 +56,13 @@ "additionalProperties": false } }, - "capability": { + "min_memory": { "type": "string", - "description": "Host capability this entry's own payload prefers, matched exactly and case-sensitively (for example metal, nvidia-cuda-12). Advisory: an unmet capability warns, it never blocks the install." + "description": "This entry's own memory requirement, for example 2GiB. One figure covers both VRAM and system RAM; the installer compares it against whichever the host will use. Advisory for the entry itself: falling short only warns." }, - "min_vram": { - "type": "string", - "description": "This entry's own VRAM floor, for example 2GiB. Positions the entry among its own candidates and must be strictly below every candidate's floor. Advisory: falling short only warns." - }, - "candidates": { + "variants": { "type": "array", - "description": "Ordered list of hardware-gated upgrades over this entry. The first candidate the host satisfies is installed, under this entry's own name; if none does, this entry installs itself. Clients that predate candidates support drop this key and install the entry unchanged.", + "description": "Other builds of the same model (other backends, other quantizations) the installer may pick instead of this entry's own payload. Order carries no meaning: the installer drops what this host cannot run or does not have the memory for, then takes the largest of what is left, installing it under this entry's own name. If nothing is left this entry installs itself. Clients that predate variants support drop this key and install the entry unchanged.", "minItems": 1, "items": { "type": "object", @@ -74,15 +70,11 @@ "properties": { "model": { "type": "string", - "description": "Name of a gallery entry that declares no candidates of its own" - }, - "capability": { - "type": "string", - "description": "Host capability this candidate requires, matched exactly and case-sensitively (for example metal, nvidia-cuda-12). Absent matches any host." + "description": "Name of a gallery entry that declares no variants of its own" }, - "min_vram": { + "min_memory": { "type": "string", - "description": "Authored VRAM floor, for example 20GiB. Authoritative: inference never overwrites it." + "description": "Authored memory requirement, for example 20GiB. Optional, and only needed to override an inferred estimate that is wrong. Authoritative: inference never overwrites it." }, "backend": { "type": "string", @@ -92,9 +84,9 @@ "type": "string", "description": "Denormalized by the nightly job for display. Never authored by hand." }, - "inferred_min_vram": { + "inferred_min_memory": { "type": "string", - "description": "Denormalized by the nightly job as a fallback floor. Never authored by hand; min_vram wins." + "description": "Denormalized by the nightly job as a fallback requirement. Never authored by hand; min_memory wins." } }, "additionalProperties": false diff --git a/gallery/index.yaml b/gallery/index.yaml index 57ebc707c633..c34b33fe1a5d 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -4296,13 +4296,14 @@ - math last_checked: "2026-04-30" # This entry installs the Q4_K_M build as-is, on any client and any host. - # min_vram positions it as its own last-resort candidate, and candidates - # lists the upgrades a bigger host gets instead. Clients that predate - # candidates support drop both keys and install this entry unchanged. - min_vram: 2GiB - candidates: + # variants lists other builds of the same model the installer may pick + # instead when this host can run them and has the memory for them. Clients + # that predate variants support drop both keys and install this entry + # unchanged. + min_memory: 2GiB + variants: - model: nanbeige4.1-3b-q8 - min_vram: 6GiB + min_memory: 6GiB overrides: parameters: model: nanbeige4.1-3b-q4_k_m.gguf From 7159c381ea998f7fe803635d247622b23859ce1a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 22:32:04 +0000 Subject: [PATCH 17/48] gallery: size model variants with a live probe, drop the nightly denormalizer Selection needs each variant's size to decide whether it fits and to rank largest-first. That figure was written into the index by a nightly job, which made the gallery carry a derived value that could drift from the entry it was derived from. Derive it at install time instead. pkg/vram already sizes a model without downloading it, and the gallery UI already uses it: a remote GGUF header range-fetch, then an HTTP HEAD for the content length, then any declared size:. It caches its results, so reuse it rather than writing a second probing path. A probe failure must never fail an install, so an unprobeable variant is treated as unknown: it survives the memory filter, because nothing proves it does not fit, and it ranks last, so a known-good fit always beats a guess. If every probe fails, selection still terminates on the base entry. The probe is injected through ResolveEnv rather than called directly, for the same reason the backend compatibility check is: specs pin an exact size, or an exact failure, without reaching the network. With that in place three things are dead weight and go: - The nightly job and the fields it populated. Variant.Backend was redundant because the backend is resolved live from the referenced entry during selection, and Quantization was display-only that nothing read. - min_memory on the base entry. The base always installs and its floor could only warn, so it could not change any outcome. - The lint rules and schema entries for both. min_memory on individual variants stays, as the override for when the probed size is wrong. An authored figure now suppresses the probe entirely rather than merely outranking it, so it costs no round trip. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- .github/ci/gallery_denormalize.go | 296 --------------------- .github/workflows/gallery-denormalize.yaml | 56 ---- core/gallery/models.go | 86 +++++- core/gallery/models_types.go | 4 - core/gallery/resolve_variant.go | 41 ++- core/gallery/resolve_variant_test.go | 110 +++++--- core/gallery/variant.go | 34 +-- core/gallery/variants_install_test.go | 109 +++++++- core/gallery/variants_lint_test.go | 66 ++--- core/schema/gallery-model.schema.json | 18 +- gallery/index.yaml | 8 +- 11 files changed, 326 insertions(+), 502 deletions(-) delete mode 100644 .github/ci/gallery_denormalize.go delete mode 100644 .github/workflows/gallery-denormalize.yaml diff --git a/.github/ci/gallery_denormalize.go b/.github/ci/gallery_denormalize.go deleted file mode 100644 index 2dd0df909aa2..000000000000 --- a/.github/ci/gallery_denormalize.go +++ /dev/null @@ -1,296 +0,0 @@ -// Command gallery_denormalize fills the read-only denormalized fields on the -// variants of gallery model entries: backend, quantization, and -// inferred_min_memory. -// -// It never modifies an authored min_memory. Authored values are authoritative, -// because a human who measured a real load knows more than a pre-download -// estimate does. -package main - -import ( - "bytes" - "context" - "fmt" - "os" - "time" - - "gopkg.in/yaml.v3" - - "github.com/mudler/LocalAI/core/gallery" - "github.com/mudler/LocalAI/pkg/vram" -) - -const estimateContext = 8192 - -func main() { - if len(os.Args) < 2 { - fmt.Fprintln(os.Stderr, "usage: gallery_denormalize ") - os.Exit(1) - } - path := os.Args[1] - - data, err := os.ReadFile(path) - if err != nil { - fmt.Fprintf(os.Stderr, "read %s: %v\n", path, err) - os.Exit(1) - } - - var entries []gallery.GalleryModel - if err := yaml.Unmarshal(data, &entries); err != nil { - fmt.Fprintf(os.Stderr, "parse %s: %v\n", path, err) - os.Exit(1) - } - - byName := map[string]gallery.GalleryModel{} - for _, e := range entries { - byName[e.Name] = e - } - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - failures := 0 - for i := range entries { - if !entries[i].HasVariants() { - continue - } - for j := range entries[i].Variants { - c := &entries[i].Variants[j] - - target, ok := byName[c.Model] - if !ok { - fmt.Fprintf(os.Stderr, "%s: variant %q not found\n", entries[i].Name, c.Model) - failures++ - continue - } - - // The referenced entry's payload usually lives behind its url - // rather than inline, so fetch it. Metadata.Backend is populated - // at load time by the running server, not by parsing the index, - // so it is always empty here and cannot be copied. - resolved, err := gallery.GetGalleryConfigFromURLWithContext[gallery.ModelConfig](ctx, target.URL, "") - if err != nil { - fmt.Fprintf(os.Stderr, "%s: cannot fetch config for %q: %v\n", entries[i].Name, c.Model, err) - failures++ - continue - } - - c.Backend = backendOf(target, resolved) - c.Quantization = quantizationOf(target) - - // An authored value wins outright, so it gets no inferred - // counterpart. Clearing first matters: a variant that gained an - // authored min_memory would otherwise keep a stale inferred figure - // that EffectiveMinMemory would go on reporting once the authored - // one is removed again. - if c.MinMemory != "" { - c.InferredMinMemory = "" - continue - } - - // Weight files can come from either side: a referenced index entry - // usually lists them itself (files:, i.e. AdditionalFiles) while - // the url it points at is only a base config, but some entries - // invert that. Install time downloads both, so estimate over both. - files := make([]vram.FileInput, 0, len(target.AdditionalFiles)+len(resolved.Files)) - for _, f := range target.AdditionalFiles { - files = append(files, vram.FileInput{URI: f.URI}) - } - for _, f := range resolved.Files { - files = append(files, vram.FileInput{URI: f.URI}) - } - - estimate, err := vram.EstimateModelMultiContext(ctx, vram.ModelEstimateInput{ - Files: files, - Size: target.Size, - }, []uint32{estimateContext}) - if err != nil || estimate.VRAMForContext(estimateContext) == 0 { - fmt.Fprintf(os.Stderr, - "%s: could not estimate min_memory for %q (%v); author one by hand\n", - entries[i].Name, c.Model, err) - failures++ - continue - } - c.InferredMinMemory = fmt.Sprintf("%dMiB", estimate.VRAMForContext(estimateContext)/(1024*1024)) - } - } - - if err := writeVariants(path, data, entries); err != nil { - fmt.Fprintf(os.Stderr, "write %s: %v\n", path, err) - os.Exit(1) - } - - if failures > 0 { - fmt.Fprintf(os.Stderr, "%d variant(s) need a hand-authored min_memory\n", failures) - os.Exit(1) - } -} - -// writeVariants writes back only the three derived keys on each variant, -// leaving every other byte of the index alone. -// -// Round-tripping through []GalleryModel would reflow all 1200+ entries -// (quoting, key order, line wrapping), burying the handful of real changes in -// reformatting noise and making the nightly PR unreviewable. Even re-encoding -// just the variants subtree would restyle authored fields such as min_memory, -// so the rewrite reaches down to individual mapping keys and the node tree is -// re-encoded at the index's authored indent. That also makes the job -// idempotent: a run that computes the same values leaves the file untouched -// and opens no PR. -func writeVariants(path string, data []byte, entries []gallery.GalleryModel) error { - var doc yaml.Node - if err := yaml.Unmarshal(data, &doc); err != nil { - return fmt.Errorf("re-parse for node rewrite: %w", err) - } - if doc.Kind != yaml.DocumentNode || len(doc.Content) == 0 { - return fmt.Errorf("unexpected document shape") - } - root := doc.Content[0] - if root.Kind != yaml.SequenceNode { - return fmt.Errorf("index root is not a sequence") - } - if len(root.Content) != len(entries) { - return fmt.Errorf("entry count drift: %d nodes vs %d entries", len(root.Content), len(entries)) - } - - changed := false - for i, entryNode := range root.Content { - if !entries[i].HasVariants() || entryNode.Kind != yaml.MappingNode { - continue - } - - variantsNode := mappingValue(entryNode, "variants") - if variantsNode == nil || variantsNode.Kind != yaml.SequenceNode { - return fmt.Errorf("entry %q has variants but no variants sequence in the document", entries[i].Name) - } - if len(variantsNode.Content) != len(entries[i].Variants) { - return fmt.Errorf("entry %q variant count drift: %d nodes vs %d parsed", - entries[i].Name, len(variantsNode.Content), len(entries[i].Variants)) - } - - for j, variantNode := range variantsNode.Content { - if variantNode.Kind != yaml.MappingNode { - return fmt.Errorf("entry %q variant %d is not a mapping", entries[i].Name, j) - } - c := entries[i].Variants[j] - for _, kv := range []struct{ key, value string }{ - {"backend", c.Backend}, - {"quantization", c.Quantization}, - {"inferred_min_memory", c.InferredMinMemory}, - } { - if setMappingValue(variantNode, kv.key, kv.value) { - changed = true - } - } - } - } - - // Nothing to rewrite means nothing to encode: leave the file, and its - // mtime, alone. - if !changed { - return nil - } - - // yaml.Marshal encodes at yaml.v3's default 4-space indent, which reflows - // every nested block in the file (~6000 lines against the real index) and - // drowns the handful of rewritten keys. The index is authored at 2, so - // encode at 2 and the diff stays limited to what actually changed. - var buf bytes.Buffer - enc := yaml.NewEncoder(&buf) - enc.SetIndent(2) - if err := enc.Encode(&doc); err != nil { - return fmt.Errorf("encode: %w", err) - } - if err := enc.Close(); err != nil { - return fmt.Errorf("flush encoder: %w", err) - } - - // The encoder does not emit a document start marker, so restore the one the - // file was authored with rather than deleting it on every run. - out := buf.Bytes() - if header := documentHeader(data); header != nil && !bytes.HasPrefix(out, header) { - out = append(header, out...) - } - - // Preserve the file's existing mode instead of forcing 0644. - info, err := os.Stat(path) - if err != nil { - return fmt.Errorf("stat: %w", err) - } - return os.WriteFile(path, out, info.Mode().Perm()) -} - -// documentHeader returns the leading document start marker line, or nil when the -// file was authored without one. -func documentHeader(data []byte) []byte { - for _, marker := range [][]byte{[]byte("---\n"), []byte("---\r\n")} { - if bytes.HasPrefix(data, marker) { - return marker - } - } - return nil -} - -// mappingValue returns the value node for key, or nil when absent. -func mappingValue(mapping *yaml.Node, key string) *yaml.Node { - for i := 0; i+1 < len(mapping.Content); i += 2 { - if mapping.Content[i].Value == key { - return mapping.Content[i+1] - } - } - return nil -} - -// setMappingValue sets key to value, appending the key when absent and dropping -// it when value is empty so a field that stops being derivable does not leave a -// stale number behind. It reports whether the document actually changed. -func setMappingValue(mapping *yaml.Node, key, value string) bool { - for i := 0; i+1 < len(mapping.Content); i += 2 { - if mapping.Content[i].Value != key { - continue - } - if value == "" { - mapping.Content = append(mapping.Content[:i], mapping.Content[i+2:]...) - return true - } - if mapping.Content[i+1].Value == value { - return false - } - mapping.Content[i+1] = scalarNode(value) - return true - } - if value == "" { - return false - } - mapping.Content = append(mapping.Content, scalarNode(key), scalarNode(value)) - return true -} - -func scalarNode(value string) *yaml.Node { - return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value} -} - -// backendOf mirrors the priority used by resolveBackend in -// core/gallery/backend_resolve.go: an entry's overrides win over whatever the -// referenced config file declares. -func backendOf(m gallery.GalleryModel, resolved gallery.ModelConfig) string { - if b, ok := m.Overrides["backend"].(string); ok && b != "" { - return b - } - var cfg struct { - Backend string `yaml:"backend"` - } - if err := yaml.Unmarshal([]byte(resolved.ConfigFile), &cfg); err == nil { - return cfg.Backend - } - return "" -} - -// quantizationOf reports the quantization declared by a referenced entry's -// overrides, empty when it declares none. -func quantizationOf(m gallery.GalleryModel) string { - if q, ok := m.Overrides["quantization"].(string); ok { - return q - } - return "" -} diff --git a/.github/workflows/gallery-denormalize.yaml b/.github/workflows/gallery-denormalize.yaml deleted file mode 100644 index e58a83e1f949..000000000000 --- a/.github/workflows/gallery-denormalize.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: Denormalize gallery meta entries -on: - schedule: - - cron: 0 22 * * * - workflow_dispatch: -jobs: - denormalize: - if: github.repository == 'mudler/LocalAI' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - # The program exits 1 when any candidate could not be estimated, but it - # still writes everything it did compute. Aborting here would let one - # unreachable candidate block the refresh of every other one, night after - # night, so record the status and let the PR open with the partial result. - - name: Denormalize meta candidates - id: denormalize - run: | - set +e - go run ./.github/ci/gallery_denormalize.go gallery/index.yaml - status=$? - set -e - echo "status=$status" >> "$GITHUB_OUTPUT" - if [ "$status" -ne 0 ]; then - echo "partial=yes" >> "$GITHUB_OUTPUT" - echo "note=**This run was partial.** At least one candidate could not be estimated, so some fields may be missing or stale. See the workflow log." >> "$GITHUB_OUTPUT" - else - echo "partial=no" >> "$GITHUB_OUTPUT" - echo "note=All candidates were processed successfully." >> "$GITHUB_OUTPUT" - fi - - name: Create Pull Request - uses: peter-evans/create-pull-request@v8 - with: - token: ${{ secrets.UPDATE_BOT_TOKEN }} - push-to-fork: ci-forks/LocalAI - commit-message: ':arrow_up: Denormalize meta model candidates' - title: 'chore(model-gallery): :arrow_up: refresh meta candidate metadata' - branch: "update/gallery-denormalize" - body: | - Refreshes the denormalized `backend`, `quantization` and - `inferred_min_vram` fields on meta model entries. - - Authored `min_vram` values are never modified. - - ${{ steps.denormalize.outputs.note }} - signoff: true - # Surface the failure only after the PR exists, so the problem stays - # visible without discarding the work that did succeed. - - name: Report estimation failures - if: steps.denormalize.outputs.partial == 'yes' - run: | - echo "gallery_denormalize exited ${{ steps.denormalize.outputs.status }}: one or more candidates need a hand-authored min_vram." - exit 1 diff --git a/core/gallery/models.go b/core/gallery/models.go index d8bf109c1868..1792d415d4ee 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -9,6 +9,7 @@ import ( "path/filepath" "slices" "strings" + "time" "dario.cat/mergo" lconfig "github.com/mudler/LocalAI/core/config" @@ -17,6 +18,7 @@ import ( "github.com/mudler/LocalAI/pkg/modelartifacts" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/LocalAI/pkg/utils" + "github.com/mudler/LocalAI/pkg/vram" "github.com/mudler/LocalAI/pkg/xsysinfo" "github.com/mudler/xlog" @@ -88,7 +90,7 @@ type PromptTemplate struct { // The base is appended rather than special-cased downstream so there is exactly // one selection path, and so an operator can pin the entry's own name to // decline an upgrade their hardware would otherwise take. -func variantOptions(models []*GalleryModel, entry *GalleryModel) ([]VariantOption, error) { +func variantOptions(models []*GalleryModel, entry *GalleryModel, env ResolveEnv) ([]VariantOption, error) { options := make([]VariantOption, 0, len(entry.Variants)+1) for _, v := range entry.Variants { // Every referenced entry is looked up here rather than lazily after @@ -101,16 +103,73 @@ func variantOptions(models []*GalleryModel, entry *GalleryModel) ([]VariantOptio if target.HasVariants() { return nil, fmt.Errorf("model %q references variant %q which declares variants of its own; resolution is a single pass, so those would be silently ignored", entry.Name, v.Model) } - options = append(options, VariantOption{Variant: v, Backend: target.Backend}) + + option := VariantOption{Variant: v, Backend: target.Backend} + // An authored figure short circuits the probe: the author already + // answered the question the probe would spend a round trip asking. + if v.MinMemory == "" && env.ProbeMemory != nil { + option.ProbedMemory = env.ProbeMemory(target) + } + options = append(options, option) } + // The base is never filtered nor ranked, so its size would change nothing + // and probing it would spend a round trip on a discarded answer. return append(options, VariantOption{ - Variant: Variant{Model: entry.Name, MinMemory: entry.MinMemory}, + Variant: Variant{Model: entry.Name}, Backend: entry.Backend, IsBase: true, }), nil } +// probeContextLength is the context size the footprint estimate is taken at. +// Selection compares whole models against whole hosts, so this only has to be a +// consistent, realistic default rather than the context a user will eventually +// configure. +const probeContextLength = 8192 + +// probeTimeout bounds a single entry's probe. An entry with several variants +// probes each of them, so an unreachable host has to give up quickly: an +// unknown size only costs a variant its ranking, whereas a stalled probe costs +// the user the whole install. +const probeTimeout = 5 * time.Second + +// probeEntryMemory measures what a gallery entry will occupy, without +// downloading it. +// +// pkg/vram does the work and caches its results across calls: it range-fetches +// the GGUF header for a real estimate, falls back to an HTTP HEAD for the +// content length, and finally to any declared size:. A HuggingFace repo listing +// is deliberately NOT used as a further fallback, unlike the gallery UI's +// estimator: a quantization entry's urls routinely point at the base model's +// repo, and summing every weight file there would overstate one variant badly +// enough to wrongly filter it out. +// +// Zero means the size could not be determined. Callers must read that as +// unknown rather than as zero, so an unreachable network downgrades a variant's +// ranking instead of failing the install. +func probeEntryMemory(ctx context.Context, entry *GalleryModel) uint64 { + input := vram.ModelEstimateInput{Size: entry.Size} + for _, f := range entry.AdditionalFiles { + if vram.IsWeightFile(f.URI) { + input.Files = append(input.Files, vram.FileInput{URI: f.URI}) + } + } + if len(input.Files) == 0 && input.Size == "" { + return 0 + } + + ctx, cancel := context.WithTimeout(ctx, probeTimeout) + defer cancel() + + estimate, err := vram.EstimateModelMultiContext(ctx, input, []uint32{probeContextLength}) + if err != nil { + xlog.Debug("Could not probe a model variant's size; treating it as unknown", "model", entry.Name, "error", err) + return 0 + } + return estimate.VRAMForContext(probeContextLength) +} + // ResolveVariant picks the gallery entry to install for a host, from the // variants an entry declares plus the entry itself, and returns it renamed to // the entry's name and carrying the entry's presentation metadata. @@ -121,12 +180,13 @@ func variantOptions(models []*GalleryModel, entry *GalleryModel) ([]VariantOptio // from the entry the user asked for, so the installed model presents as that // model rather than as one of its variants. // -// The base entry always resolves. When even its own requirement falls short -// this installs it regardless, because the entry is a complete entry that every -// older LocalAI release installs unconditionally; refusing it here would make -// the gallery behave worse the newer the client is. +// The base entry always resolves, whatever the host has. It is a complete entry +// that every older LocalAI release installs unconditionally, so refusing it here +// would make the gallery behave worse the newer the client is. That is also why +// the base declares no memory requirement of its own: a floor that can only +// warn cannot change any outcome. func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, pin string) (*GalleryModel, Variant, error) { - options, err := variantOptions(models, entry) + options, err := variantOptions(models, entry, env) if err != nil { return nil, Variant{}, err } @@ -146,7 +206,7 @@ func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, // checks, but a silent bypass makes a later out-of-memory failure // impossible to trace back to the pin, so it is recorded loudly here. if pin != "" { - if need, known, verr := selected.Variant.EffectiveMinMemory(); verr == nil && known && env.AvailableMemory < need { + if need, known, verr := selected.EffectiveMemory(); verr == nil && known && env.AvailableMemory < need { xlog.Warn("Pinned model variant declares more memory than this system reports; installing anyway because the pin overrides hardware resolution", "model", entry.Name, "variant", selected.Variant.Model, "required_memory", need, "available_memory", env.AvailableMemory) } @@ -166,10 +226,9 @@ func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, resolved.Icon = entry.Icon resolved.License = entry.License // The resolved entry is a concrete install target, so it must not carry the - // selection fields any more; leaving them would let a second pass resolve - // the already-resolved entry all over again. + // variant list any more; leaving it would let a second pass resolve the + // already-resolved entry all over again. resolved.Variants = nil - resolved.MinMemory = "" // The struct copy above is shallow, so every reference-typed field still // aliases the gallery's own entries. The install path mutates Overrides in @@ -396,6 +455,9 @@ func InstallModelFromGallery( BackendCompatible: func(backend string) bool { return systemState.IsBackendCompatible(backend, "") }, + ProbeMemory: func(target *GalleryModel) uint64 { + return probeEntryMemory(ctx, target) + }, } resolved, variant, err := ResolveVariant(models, model, env, pin) diff --git a/core/gallery/models_types.go b/core/gallery/models_types.go index 2a48ca68421c..922ac170381b 100644 --- a/core/gallery/models_types.go +++ b/core/gallery/models_types.go @@ -25,10 +25,6 @@ type GalleryModel struct { // stays a complete, installable entry and older LocalAI releases, which drop // this key, install it exactly as before. Variants []Variant `json:"variants,omitempty" yaml:"variants,omitempty"` - // MinMemory is this entry's own memory requirement (e.g. "2GiB"), in the - // same form as a variant's. It is advisory for the base entry: falling short - // only warns, because the base always installs. - MinMemory string `json:"min_memory,omitempty" yaml:"min_memory,omitempty"` } func (m *GalleryModel) GetInstalled() bool { diff --git a/core/gallery/resolve_variant.go b/core/gallery/resolve_variant.go index cdfceeae6283..cb111bc61c86 100644 --- a/core/gallery/resolve_variant.go +++ b/core/gallery/resolve_variant.go @@ -30,6 +30,29 @@ type VariantOption struct { // filter and is the answer when nothing else survives, because the entry // must stay installable on every host and for every client. IsBase bool + // ProbedMemory is the footprint measured live from the referenced entry's + // weights, in bytes. It is only consulted when the variant declares no + // min_memory of its own, because a human who measured a real load knows + // more than a pre-download estimate does. + // + // Zero means the probe could not determine a size. That is an unknown, not + // a zero requirement: a probe that cannot reach the network must never be + // able to break an install. + ProbedMemory uint64 +} + +// EffectiveMemory returns this option's memory requirement in bytes and whether +// one is known at all: the authored figure when there is one, else the live +// probe result, else nothing. +func (o VariantOption) EffectiveMemory() (uint64, bool, error) { + size, known, err := o.Variant.AuthoredMinMemory() + if err != nil || known { + return size, known, err + } + if o.ProbedMemory > 0 { + return o.ProbedMemory, true, nil + } + return 0, false, nil } // ResolveEnv describes the host a variant is selected for. @@ -45,6 +68,18 @@ type ResolveEnv struct { // A nil func treats every backend as runnable, the right default for a // caller with no view of the hardware. BackendCompatible func(backend string) bool + // ProbeMemory measures how much memory a referenced gallery entry needs, + // without downloading it. It is consulted only for variants that declare no + // min_memory, and a zero result means "could not tell", never "needs + // nothing". + // + // It is a func field rather than a live network handle so specs can pin an + // exact size, or an exact failure, without reaching the internet. A nil func + // leaves every unauthored variant unknown, which selection already handles. + // + // SelectVariant never calls this: the install layer resolves every size into + // VariantOption.ProbedMemory first, so the selector stays pure. + ProbeMemory func(entry *GalleryModel) uint64 } func (e ResolveEnv) backendRuns(backend string) bool { @@ -75,8 +110,8 @@ type VariantSelection struct { // the hardware gate, derived from the backend name. // 3. Variants whose known memory requirement exceeds what the host has are // dropped. A variant with an UNKNOWN requirement survives, because nothing -// proves it does not fit and refusing on a missing estimate would punish -// the entries the denormalizer has not reached yet. +// proves it does not fit and refusing on a size the probe could not read +// would let a network hiccup silently downgrade what gets installed. // 4. The largest survivor wins. A bigger footprint is a higher quality // quantization of the same model, so among things that fit, more is better. // Unknown requirements rank last, so a proven fit always beats a guess. @@ -114,7 +149,7 @@ func SelectVariant(options []VariantOption, env ResolveEnv, pin string) (Variant continue } - memory, known, err := o.Variant.EffectiveMinMemory() + memory, known, err := o.EffectiveMemory() if err != nil { return VariantSelection{}, err } diff --git a/core/gallery/resolve_variant_test.go b/core/gallery/resolve_variant_test.go index 190fe3c1e90c..49fcb3468ac5 100644 --- a/core/gallery/resolve_variant_test.go +++ b/core/gallery/resolve_variant_test.go @@ -7,34 +7,49 @@ import ( "github.com/mudler/LocalAI/core/gallery" ) -var _ = Describe("Variant.EffectiveMinMemory", func() { - It("reports no requirement when neither authored nor inferred", func() { - v := gallery.Variant{Model: "x"} - size, known, err := v.EffectiveMinMemory() +var _ = Describe("VariantOption.EffectiveMemory", func() { + It("reports no requirement when nothing is authored and nothing was probed", func() { + o := gallery.VariantOption{Variant: gallery.Variant{Model: "x"}} + size, known, err := o.EffectiveMemory() Expect(err).ToNot(HaveOccurred()) Expect(known).To(BeFalse()) Expect(size).To(Equal(uint64(0))) }) - It("uses the inferred value when nothing is authored", func() { - v := gallery.Variant{Model: "x", InferredMinMemory: "6GiB"} - size, known, err := v.EffectiveMinMemory() + It("uses the probed size when nothing is authored", func() { + o := gallery.VariantOption{Variant: gallery.Variant{Model: "x"}, ProbedMemory: 6 * 1024 * 1024 * 1024} + size, known, err := o.EffectiveMemory() Expect(err).ToNot(HaveOccurred()) Expect(known).To(BeTrue()) Expect(size).To(Equal(uint64(6 * 1024 * 1024 * 1024))) }) - It("lets an authored value win over an inferred one", func() { - v := gallery.Variant{Model: "x", MinMemory: "20GiB", InferredMinMemory: "6GiB"} - size, known, err := v.EffectiveMinMemory() + It("lets an authored figure win over a probed one", func() { + // A human who measured a real load knows more than a pre-download + // estimate does, which is the entire reason min_memory survives. + o := gallery.VariantOption{ + Variant: gallery.Variant{Model: "x", MinMemory: "20GiB"}, + ProbedMemory: 6 * 1024 * 1024 * 1024, + } + size, known, err := o.EffectiveMemory() Expect(err).ToNot(HaveOccurred()) Expect(known).To(BeTrue()) Expect(size).To(Equal(uint64(20 * 1024 * 1024 * 1024))) }) - It("errors on an unparseable figure rather than silently treating it as absent", func() { - v := gallery.Variant{Model: "x", MinMemory: "twenty gigs"} - _, _, err := v.EffectiveMinMemory() + It("treats a failed probe as unknown rather than as a zero requirement", func() { + // A probe that could not reach the network reports 0. Reading that as + // "needs nothing" would make an unreachable host look like the perfect + // fit and hand the user the largest download on offer. + o := gallery.VariantOption{Variant: gallery.Variant{Model: "x"}, ProbedMemory: 0} + _, known, err := o.EffectiveMemory() + Expect(err).ToNot(HaveOccurred()) + Expect(known).To(BeFalse()) + }) + + It("errors on an unparseable authored figure rather than silently ignoring it", func() { + o := gallery.VariantOption{Variant: gallery.Variant{Model: "x", MinMemory: "twenty gigs"}} + _, _, err := o.EffectiveMemory() Expect(err).To(HaveOccurred()) }) }) @@ -49,9 +64,13 @@ var _ = Describe("SelectVariant", func() { } } - base := func(model, minMemory string) gallery.VariantOption { - o := option(model, "llama-cpp", minMemory) + // The base entry declares no memory requirement of its own, so a base + // option's size can only ever come from a probe. It is exempt from every + // filter regardless, which the fallback specs below pin down. + base := func(model string, probed uint64) gallery.VariantOption { + o := option(model, "llama-cpp", "") o.IsBase = true + o.ProbedMemory = probed return o } @@ -68,7 +87,7 @@ var _ = Describe("SelectVariant", func() { options := []gallery.VariantOption{ option("m-mlx-8bit", "mlx", "24GiB"), option("m-gguf-q8", "llama-cpp", "12GiB"), - base("m-gguf-q4", "6GiB"), + base("m-gguf-q4", gib(6)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ @@ -85,7 +104,7 @@ var _ = Describe("SelectVariant", func() { options := []gallery.VariantOption{ option("m-mlx-8bit", "mlx", "24GiB"), option("m-gguf-q8", "llama-cpp", "12GiB"), - base("m-gguf-q4", "6GiB"), + base("m-gguf-q4", gib(6)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ @@ -97,7 +116,7 @@ var _ = Describe("SelectVariant", func() { }) It("treats every backend as runnable when the host cannot be inspected", func() { - options := []gallery.VariantOption{option("m-mlx-8bit", "mlx", "24GiB"), base("m-gguf-q4", "6GiB")} + options := []gallery.VariantOption{option("m-mlx-8bit", "mlx", "24GiB"), base("m-gguf-q4", gib(6))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(80)}, "") Expect(err).ToNot(HaveOccurred()) @@ -113,7 +132,7 @@ var _ = Describe("SelectVariant", func() { option("m-q4", "llama-cpp", "6GiB"), option("m-q8", "llama-cpp", "12GiB"), option("m-f16", "llama-cpp", "24GiB"), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "") @@ -129,7 +148,7 @@ var _ = Describe("SelectVariant", func() { option("m-f16", "llama-cpp", "24GiB"), option("m-q8", "llama-cpp", "12GiB"), option("m-q4", "llama-cpp", "6GiB"), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "") @@ -144,7 +163,7 @@ var _ = Describe("SelectVariant", func() { options := []gallery.VariantOption{ option("m-unknown", "llama-cpp", ""), option("m-q8", "llama-cpp", "12GiB"), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "") @@ -155,7 +174,7 @@ var _ = Describe("SelectVariant", func() { It("keeps a variant of unknown size rather than dropping it", func() { options := []gallery.VariantOption{ option("m-unknown", "llama-cpp", ""), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(4)}, "") @@ -164,8 +183,30 @@ var _ = Describe("SelectVariant", func() { Expect(selection.FellBackToBase).To(BeFalse()) }) + It("ranks and filters a probed size exactly as an authored one", func() { + // Nothing here declares min_memory, so every figure came from the + // live probe. The probe is the primary source now and authoring the + // exception, so it has to drive both the filter and the ranking. + probed := func(model string, size uint64) gallery.VariantOption { + o := option(model, "llama-cpp", "") + o.ProbedMemory = size + return o + } + options := []gallery.VariantOption{ + probed("m-q4", gib(6)), + probed("m-f16", gib(24)), + probed("m-q8", gib(12)), + base("m-base", gib(2)), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-q8")) + Expect(selection.Reasons).To(ContainElement(ContainSubstring("m-f16"))) + }) + It("admits a variant needing exactly the memory available", func() { - options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", "2GiB")} + options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", gib(2))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(12)}, "") Expect(err).ToNot(HaveOccurred()) @@ -178,7 +219,7 @@ var _ = Describe("SelectVariant", func() { options := []gallery.VariantOption{ option("m-q8", "llama-cpp", "12GiB"), option("m-f16", "llama-cpp", "24GiB"), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(4)}, "") @@ -189,10 +230,11 @@ var _ = Describe("SelectVariant", func() { Expect(selection.Reasons).To(HaveLen(2)) }) - It("selects the base even when the base's own requirement is unmet", func() { - // There is nothing below the base, so refusing here would make an - // entry every older client installs fine uninstallable on newer ones. - options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", "2GiB")} + It("selects the base even when the base does not fit either", func() { + // The base is exempt from the memory filter, not merely favoured by + // it: there is nothing below it, so refusing here would make an entry + // every older client installs fine uninstallable on newer ones. + options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", gib(2))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: 0}, "") Expect(err).ToNot(HaveOccurred()) @@ -203,7 +245,7 @@ var _ = Describe("SelectVariant", func() { options := []gallery.VariantOption{ option("m-mlx", "mlx", "8GiB"), option("m-f16", "llama-cpp", "24GiB"), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ @@ -227,7 +269,7 @@ var _ = Describe("SelectVariant", func() { It("honors a pin the hardware would never have chosen", func() { options := []gallery.VariantOption{ option("m-mlx", "mlx", "64GiB"), - base("m-base", "2GiB"), + base("m-base", gib(2)), } selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{ @@ -239,7 +281,7 @@ var _ = Describe("SelectVariant", func() { }) It("honors a pin naming the base, declining an upgrade that fits", func() { - options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", "2GiB")} + options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", gib(2))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "m-base") Expect(err).ToNot(HaveOccurred()) @@ -247,7 +289,7 @@ var _ = Describe("SelectVariant", func() { }) It("matches a pin case-insensitively", func() { - options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", "2GiB")} + options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", gib(2))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "M-F16") Expect(err).ToNot(HaveOccurred()) @@ -255,7 +297,7 @@ var _ = Describe("SelectVariant", func() { }) It("fails loudly when the pin names nothing in the list", func() { - options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", "2GiB")} + options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", gib(2))} _, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "m-gone") Expect(err).To(MatchError(gallery.ErrPinNotFound)) @@ -264,7 +306,7 @@ var _ = Describe("SelectVariant", func() { }) It("propagates an unparseable figure instead of treating it as unconstrained", func() { - options := []gallery.VariantOption{option("bad", "llama-cpp", "lots"), base("m-base", "2GiB")} + options := []gallery.VariantOption{option("bad", "llama-cpp", "lots"), base("m-base", gib(2))} _, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(8)}, "") Expect(err).To(HaveOccurred()) diff --git a/core/gallery/variant.go b/core/gallery/variant.go index 0821f440a4c4..35ca8bfb7b93 100644 --- a/core/gallery/variant.go +++ b/core/gallery/variant.go @@ -16,41 +16,29 @@ type Variant struct { // Model is the name of a gallery entry that declares no variants of its own. Model string `json:"model" yaml:"model"` // MinMemory is the authored memory requirement (e.g. "20GiB"). It exists to - // override an inferred estimate that is known to be wrong; most variants - // should not need it. + // override the size the installer probes live from the model's weights when + // that figure is known to be wrong; most variants should not need it. // // One number covers both VRAM and system RAM. A model's footprint is // roughly the same wherever it lives, and the selector compares this against // whichever of the two this host will actually use, so splitting it in two // would only invite the pair to disagree. MinMemory string `json:"min_memory,omitempty" yaml:"min_memory,omitempty"` - - // The fields below are denormalized by the nightly job for display and - // lint. They are never authored by hand and never affect what gets - // installed, because installation reads the referenced entry live. - Backend string `json:"backend,omitempty" yaml:"backend,omitempty"` - Quantization string `json:"quantization,omitempty" yaml:"quantization,omitempty"` - InferredMinMemory string `json:"inferred_min_memory,omitempty" yaml:"inferred_min_memory,omitempty"` } -// EffectiveMinMemory returns this variant's memory requirement in bytes and -// whether one is known at all. An authored MinMemory wins over an inferred one, -// because a human who measured a real load knows more than a pre-download -// estimate does. +// AuthoredMinMemory returns the hand-written memory requirement in bytes and +// whether one was written at all. // -// An unknown requirement is not a zero requirement. Callers must not read the -// returned 0 as "fits anywhere"; it means nothing is known either way. -func (v Variant) EffectiveMinMemory() (uint64, bool, error) { - raw := v.MinMemory - if raw == "" { - raw = v.InferredMinMemory - } - if raw == "" { +// An absent requirement is not a zero requirement. Callers must not read the +// returned 0 as "fits anywhere"; it means the author said nothing, and the +// figure has to come from a live probe instead. +func (v Variant) AuthoredMinMemory() (uint64, bool, error) { + if v.MinMemory == "" { return 0, false, nil } - size, err := vram.ParseSizeString(raw) + size, err := vram.ParseSizeString(v.MinMemory) if err != nil { - return 0, false, fmt.Errorf("variant %q has an unparseable min_memory %q: %w", v.Model, raw, err) + return 0, false, fmt.Errorf("variant %q has an unparseable min_memory %q: %w", v.Model, v.MinMemory, err) } return size, true, nil } diff --git a/core/gallery/variants_install_test.go b/core/gallery/variants_install_test.go index 1de4e5ace432..424bbfd81f7f 100644 --- a/core/gallery/variants_install_test.go +++ b/core/gallery/variants_install_test.go @@ -28,12 +28,11 @@ var _ = Describe("GalleryModel variant declarations", func() { Expect(m.HasVariants()).To(BeTrue()) }) - It("parses an entry's own memory figure and its variant list", func() { + It("parses a variant list", func() { var m gallery.GalleryModel err := yaml.Unmarshal([]byte(` name: qwen3.6-27b url: "github:example/repo/qwen3.6-27b.yaml@master" -min_memory: 4GiB variants: - model: qwen3.6-27b-mlx-8bit - model: qwen3.6-27b-gguf-q8 @@ -42,7 +41,6 @@ variants: Expect(err).ToNot(HaveOccurred()) Expect(m.Name).To(Equal("qwen3.6-27b")) Expect(m.URL).To(Equal("github:example/repo/qwen3.6-27b.yaml@master")) - Expect(m.MinMemory).To(Equal("4GiB")) Expect(m.HasVariants()).To(BeTrue()) Expect(m.Variants).To(HaveLen(2)) // The first variant is nothing but a name, which is the shape authoring @@ -81,7 +79,6 @@ var _ = Describe("ResolveVariant", func() { base = newModel("qwen3-8b-gguf-q4", "file://gguf.yaml", "Qwen3 8B Q4", "qwen.png") base.Backend = "llama-cpp" base.Tags = []string{"llm"} - base.MinMemory = "6GiB" base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq", MinMemory: "20GiB"}} models = []*gallery.GalleryModel{upgrade, base} }) @@ -118,20 +115,96 @@ var _ = Describe("ResolveVariant", func() { Expect(resolved.URL).To(Equal("file://gguf.yaml")) }) - It("installs the entry even when the host misses the entry's own requirement", func() { - resolved, variant, err := gallery.ResolveVariant(models, base, env(gib(1)), "") + It("installs the entry on a host with no memory budget at all", func() { + // The base carries no floor of its own precisely so this can never + // refuse; an unreadable host reports 0 and must still get the entry. + resolved, variant, err := gallery.ResolveVariant(models, base, env(0), "") Expect(err).ToNot(HaveOccurred()) Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4")) Expect(resolved.URL).To(Equal("file://gguf.yaml")) }) - It("strips the selection fields from the resolved entry", func() { - // A resolved entry is a concrete install target. Leaving the fields on - // it would let a second selection pass fire on an already-resolved entry. + It("strips the variant list from the resolved entry", func() { + // A resolved entry is a concrete install target. Leaving the list on it + // would let a second selection pass fire on an already-resolved entry. resolved, _, err := gallery.ResolveVariant(models, base, env(gib(8)), "") Expect(err).ToNot(HaveOccurred()) Expect(resolved.HasVariants()).To(BeFalse()) - Expect(resolved.MinMemory).To(BeEmpty()) + }) + + Describe("the live size probe", func() { + // The probe is injected rather than performed, so these specs pin an + // exact size, or an exact failure, without reaching the network. + probing := func(memory uint64, sizes map[string]uint64) gallery.ResolveEnv { + e := env(memory) + e.ProbeMemory = func(target *gallery.GalleryModel) uint64 { return sizes[target.Name] } + return e + } + + BeforeEach(func() { + // No authored figure anywhere, so every size below comes from the + // probe. This is the shape authoring is meant to reach for. + base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq"}} + }) + + It("selects a variant the probe shows to fit", func() { + resolved, variant, err := gallery.ResolveVariant(models, base, + probing(gib(24), map[string]uint64{"qwen3-8b-vllm-awq": gib(20)}), "") + Expect(err).ToNot(HaveOccurred()) + Expect(variant.Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(resolved.URL).To(Equal("file://vllm.yaml")) + }) + + It("rejects a variant the probe shows to be too large", func() { + // Same variant, same host, only the probed size differs, so nothing + // but the probe can account for the different answer. + resolved, variant, err := gallery.ResolveVariant(models, base, + probing(gib(24), map[string]uint64{"qwen3-8b-vllm-awq": gib(80)}), "") + Expect(err).ToNot(HaveOccurred()) + Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) + }) + + It("completes the install when the probe cannot determine a size", func() { + // A failed probe reports 0. That is an unknown, so the variant is + // not filtered out, and above all the resolution does not fail: a + // network hiccup must never be able to break an install. + resolved, variant, err := gallery.ResolveVariant(models, base, + probing(gib(24), map[string]uint64{}), "") + Expect(err).ToNot(HaveOccurred()) + Expect(variant.Model).To(Equal("qwen3-8b-vllm-awq")) + Expect(resolved.URL).To(Equal("file://vllm.yaml")) + }) + + It("still installs the base when every probe fails and nothing else survives", func() { + // The one variant this host could otherwise take is ruled out by its + // backend and no probe answers, so selection has to terminate on the + // base rather than on an error. + e := probing(gib(24), map[string]uint64{}) + e.BackendCompatible = func(backend string) bool { return backend != "vllm" } + + resolved, variant, err := gallery.ResolveVariant(models, base, e, "") + Expect(err).ToNot(HaveOccurred()) + Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) + }) + + It("does not probe a variant that declares its own min_memory", func() { + // Probing costs a network round trip, so an authored figure has to + // suppress it outright rather than merely outrank it. + base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq", MinMemory: "20GiB"}} + probed := []string{} + e := env(gib(24)) + e.ProbeMemory = func(target *gallery.GalleryModel) uint64 { + probed = append(probed, target.Name) + return gib(80) + } + + _, variant, err := gallery.ResolveVariant(models, base, e, "") + Expect(err).ToNot(HaveOccurred()) + Expect(probed).To(BeEmpty()) + Expect(variant.Model).To(Equal("qwen3-8b-vllm-awq")) + }) }) It("presents the entry's metadata, not the variant's", func() { @@ -360,6 +433,22 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { Expect(os.IsNotExist(err)).To(BeTrue(), "the variant must not be installed under its own name") }) + It("completes the install when a variant's size cannot be determined", func() { + // This runs the REAL probe against an entry that declares no size and no + // weight files, which is exactly the shape a probe cannot answer for. + // It must yield an unknown, not an error: the variant survives the + // filter and the install completes, on the variant or on the base, but + // never on a failure. Nothing here touches the network. + newGallery( + withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-q8"}), + entry("qwen3-8b-q8", "upgrade-backend"), + ) + + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + Expect(installedBackend("qwen3-8b-q4")).To(BeElementOf("base-backend", "upgrade-backend")) + }) + It("installs the largest fitting variant, not the first authored", func() { // Authored smallest-first with all three within reach, so first-match // would take the small one. diff --git a/core/gallery/variants_lint_test.go b/core/gallery/variants_lint_test.go index 4d21d17419a3..a60de7783861 100644 --- a/core/gallery/variants_lint_test.go +++ b/core/gallery/variants_lint_test.go @@ -75,12 +75,12 @@ func checkVariantReferences(entries []gallery.GalleryModel) []variantViolation { return violations } -// checkVariantMemory verifies every declared memory figure parses, on the -// variants and on the entry itself. +// checkVariantMemory verifies every authored memory figure parses. // -// Absence is fine and deliberately not flagged: an unknown requirement is a -// legitimate state that selection handles by ranking the variant last. An -// UNPARSEABLE one is different, because it makes selection fail outright for +// Absence is fine and deliberately not flagged: a variant that declares nothing +// gets its size from a live probe, and even a probe that fails leaves a +// legitimate unknown that selection handles by ranking the variant last. An +// UNPARSEABLE figure is different, because it makes selection fail outright for // the whole entry rather than degrade. func checkVariantMemory(entries []gallery.GalleryModel) []variantViolation { var violations []variantViolation @@ -88,12 +88,8 @@ func checkVariantMemory(entries []gallery.GalleryModel) []variantViolation { if !e.HasVariants() { continue } - base := gallery.Variant{Model: e.Name, MinMemory: e.MinMemory} - if _, _, err := base.EffectiveMinMemory(); err != nil { - violations = append(violations, variantViolation{Entry: e.Name, Detail: "has a bad min_memory: " + err.Error()}) - } for _, v := range e.Variants { - if _, _, err := v.EffectiveMinMemory(); err != nil { + if _, _, err := v.AuthoredMinMemory(); err != nil { violations = append(violations, variantViolation{Entry: e.Name, Variant: v.Model, Detail: "has a bad min_memory: " + err.Error()}) } } @@ -123,8 +119,8 @@ func plainEntry(name, url string) gallery.GalleryModel { return e } -func entryWithVariants(name, url, minMemory string, variants ...gallery.Variant) gallery.GalleryModel { - e := gallery.GalleryModel{Variants: variants, MinMemory: minMemory} +func entryWithVariants(name, url string, variants ...gallery.Variant) gallery.GalleryModel { + e := gallery.GalleryModel{Variants: variants} e.Name = name e.URL = url return e @@ -144,13 +140,13 @@ var _ = Describe("gallery variant lint helpers", func() { // well-meaning alignment of the two would make every helper skip every // entry and pass silently. Assert the distinction directly. It("treats an entry with variants as such even though it has a url", func() { - Expect(entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "big"}).HasVariants()).To(BeTrue()) + Expect(entryWithVariants("base", "u://base", gallery.Variant{Model: "big"}).HasVariants()).To(BeTrue()) Expect(plainEntry("big", "u://big").HasVariants()).To(BeFalse()) }) It("passes every invariant on a valid entry", func() { entries := variantFixture( - entryWithVariants("base", "u://base", "4GiB", + entryWithVariants("base", "u://base", gallery.Variant{Model: "big", MinMemory: "24GiB"}, gallery.Variant{Model: "mid", MinMemory: "12GiB"}, gallery.Variant{Model: "metal-big", MinMemory: "32GiB"}, @@ -167,10 +163,10 @@ var _ = Describe("gallery variant lint helpers", func() { }) It("passes every invariant on an entry that declares no memory at all", func() { - // Authoring is meant to be nothing but a list of names, so an entry and - // its variants carrying no figures must be entirely valid. + // Authoring is meant to be nothing but a list of names, so an entry + // whose variants carry no figures must be entirely valid. entries := variantFixture( - entryWithVariants("base", "u://base", "", gallery.Variant{Model: "big"}), + entryWithVariants("base", "u://base", gallery.Variant{Model: "big"}), plainEntry("big", "u://big"), ) @@ -181,7 +177,7 @@ var _ = Describe("gallery variant lint helpers", func() { Describe("checkVariantReferences", func() { It("flags a variant naming an entry that does not exist", func() { entries := variantFixture( - entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "ghost"}), + entryWithVariants("base", "u://base", gallery.Variant{Model: "ghost"}), plainEntry("a", "u://a"), ) @@ -193,8 +189,8 @@ var _ = Describe("gallery variant lint helpers", func() { It("flags a variant naming an entry that declares variants itself", func() { entries := variantFixture( - entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "nested"}), - entryWithVariants("nested", "u://nested", "4GiB", gallery.Variant{Model: "a"}), + entryWithVariants("base", "u://base", gallery.Variant{Model: "nested"}), + entryWithVariants("nested", "u://nested", gallery.Variant{Model: "a"}), plainEntry("a", "u://a"), ) @@ -206,7 +202,7 @@ var _ = Describe("gallery variant lint helpers", func() { It("reports every breach in one pass rather than stopping at the first", func() { entries := variantFixture( - entryWithVariants("base", "u://base", "2GiB", + entryWithVariants("base", "u://base", gallery.Variant{Model: "ghost"}, gallery.Variant{Model: "phantom"}, ), @@ -219,7 +215,7 @@ var _ = Describe("gallery variant lint helpers", func() { Describe("checkVariantMemory", func() { It("flags a variant whose memory figure cannot be parsed", func() { entries := variantFixture( - entryWithVariants("base", "u://base", "2GiB", gallery.Variant{Model: "a", MinMemory: "lots"}), + entryWithVariants("base", "u://base", gallery.Variant{Model: "a", MinMemory: "lots"}), plainEntry("a", "u://a"), ) @@ -229,30 +225,16 @@ var _ = Describe("gallery variant lint helpers", func() { Expect(violations[0].Detail).To(ContainSubstring("bad min_memory")) }) - It("flags an entry whose own memory figure cannot be parsed", func() { + It("accepts a variant that declares no figure at all", func() { + // The overwhelmingly common shape: a bare name, whose size the + // installer probes live. Flagging it would make the rule reject the + // authoring style the design exists to enable. entries := variantFixture( - entryWithVariants("base", "u://base", "eight gigs", gallery.Variant{Model: "a"}), + entryWithVariants("base", "u://base", gallery.Variant{Model: "a"}), plainEntry("a", "u://a"), ) - violations := checkVariantMemory(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Entry).To(Equal("base")) - Expect(violations[0].Variant).To(BeEmpty()) - Expect(violations[0].Detail).To(ContainSubstring("bad min_memory")) - }) - - It("flags an unparseable denormalized figure just as an authored one", func() { - // The nightly job writes inferred_min_memory, so a bug there breaks - // selection exactly as a bad hand-authored value does. - entries := variantFixture( - entryWithVariants("base", "u://base", "", gallery.Variant{Model: "a", InferredMinMemory: "8 gigabytes"}), - plainEntry("a", "u://a"), - ) - - violations := checkVariantMemory(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Variant).To(Equal("a")) + Expect(checkVariantMemory(entries)).To(BeEmpty()) }) }) }) diff --git a/core/schema/gallery-model.schema.json b/core/schema/gallery-model.schema.json index 998da2ec307d..0cf0184d1335 100644 --- a/core/schema/gallery-model.schema.json +++ b/core/schema/gallery-model.schema.json @@ -56,10 +56,6 @@ "additionalProperties": false } }, - "min_memory": { - "type": "string", - "description": "This entry's own memory requirement, for example 2GiB. One figure covers both VRAM and system RAM; the installer compares it against whichever the host will use. Advisory for the entry itself: falling short only warns." - }, "variants": { "type": "array", "description": "Other builds of the same model (other backends, other quantizations) the installer may pick instead of this entry's own payload. Order carries no meaning: the installer drops what this host cannot run or does not have the memory for, then takes the largest of what is left, installing it under this entry's own name. If nothing is left this entry installs itself. Clients that predate variants support drop this key and install the entry unchanged.", @@ -74,19 +70,7 @@ }, "min_memory": { "type": "string", - "description": "Authored memory requirement, for example 20GiB. Optional, and only needed to override an inferred estimate that is wrong. Authoritative: inference never overwrites it." - }, - "backend": { - "type": "string", - "description": "Denormalized by the nightly job for display. Never authored by hand." - }, - "quantization": { - "type": "string", - "description": "Denormalized by the nightly job for display. Never authored by hand." - }, - "inferred_min_memory": { - "type": "string", - "description": "Denormalized by the nightly job as a fallback requirement. Never authored by hand; min_memory wins." + "description": "Authored memory requirement, for example 20GiB. Optional, and only needed to override the size the installer probes live from the model's weights when that figure is wrong. Authoritative: an authored figure suppresses the probe entirely." } }, "additionalProperties": false diff --git a/gallery/index.yaml b/gallery/index.yaml index c34b33fe1a5d..a91089bfb3e5 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -4297,13 +4297,11 @@ last_checked: "2026-04-30" # This entry installs the Q4_K_M build as-is, on any client and any host. # variants lists other builds of the same model the installer may pick - # instead when this host can run them and has the memory for them. Clients - # that predate variants support drop both keys and install this entry - # unchanged. - min_memory: 2GiB + # instead when this host can run them and has the memory for them, sizing + # each one live from its weights. Clients that predate variants support drop + # the key and install this entry unchanged. variants: - model: nanbeige4.1-3b-q8 - min_memory: 6GiB overrides: parameters: model: nanbeige4.1-3b-q4_k_m.gguf From 77d01de6ffd2ece1bcdab96719385001ec12c3a9 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 23:06:17 +0000 Subject: [PATCH 18/48] feat(gallery): expose model variants for selection over API, CLI and MCP A gallery entry may carry `variants:`, alternative builds of the same model. Selection already worked at install time, but nothing could see what an entry offered or ask for a specific build, so the feature was undrivable. Listing: `GET /api/models` now reports `variants` and `auto_variant` for the entries that declare variants. Each variant carries its resolved backend, its measured size and whether it fits this host. `auto_variant` is what installing without a choice would pick right now. The new gallery.DescribeVariants runs the same variantOptions + SelectVariant pass the installer runs, so the reported default cannot drift from what installing actually does, and HostResolveEnv is extracted so both derive the host and share pkg/vram's probe cache from one place. Performance: an entry that declares no variants returns early without touching the probe, so the ~1280 ordinary entries cost exactly what they cost before. Selection: `variant` is accepted on POST /models/apply, as a query param on POST /api/models/install/:id, on the gallery apply file/string request, as `local-ai models install --variant`, and as a parameter on the install_model MCP tool (both the httpapi and inproc clients). Empty means auto-select. An unknown variant name now fails the install naming what was requested. This closes a real hole: an entry declaring no variants short-circuits before selection runs, so a requested variant was previously dropped silently and the install reported success. startup.InstallModels ends in a variadic model list, so install options could not be appended to it; InstallModelsWithOptions is added alongside and InstallModels delegates to it. No caller signature changed. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- .agents/adding-gallery-models.md | 40 +++ core/cli/models.go | 7 +- core/gallery/describe_variants.go | 85 ++++++ core/gallery/describe_variants_test.go | 247 ++++++++++++++++++ core/gallery/models.go | 48 ++-- core/gallery/variants_install_test.go | 30 +++ core/http/endpoints/localai/gallery.go | 6 + core/http/routes/ui_api.go | 32 ++- core/services/galleryop/managers_local.go | 6 +- core/services/galleryop/model_variant_test.go | 102 ++++++++ core/services/galleryop/models.go | 13 +- core/services/galleryop/operation.go | 9 + core/startup/model_preload.go | 11 +- docs/content/features/model-gallery.md | 64 +++++ pkg/mcp/localaitools/dto.go | 1 + pkg/mcp/localaitools/httpapi/client.go | 3 + pkg/mcp/localaitools/httpapi/client_test.go | 33 +++ pkg/mcp/localaitools/inproc/client.go | 1 + .../prompts/skills/install_chat_model.md | 2 +- pkg/mcp/localaitools/tools_models.go | 2 +- 20 files changed, 719 insertions(+), 23 deletions(-) create mode 100644 core/gallery/describe_variants.go create mode 100644 core/gallery/describe_variants_test.go create mode 100644 core/services/galleryop/model_variant_test.go diff --git a/.agents/adding-gallery-models.md b/.agents/adding-gallery-models.md index fe5a57e87177..9e44b0a118f3 100644 --- a/.agents/adding-gallery-models.md +++ b/.agents/adding-gallery-models.md @@ -91,6 +91,46 @@ To add a variant (e.g., different quantization), use YAML merge: uri: huggingface:////-Q8_0.gguf ``` +## Offering several builds of one model (`variants`) + +When the same model is published in more than one quantization, or is also +servable by another engine, add each build as its own ordinary gallery entry and +then point one of them at the others with `variants`: + +```yaml +- !!merge <<: *chatml + name: "nanbeige4.1-3b-q4" + # ... the usual urls / overrides / files for the Q4 build ... + variants: + - model: nanbeige4.1-3b-q8 +``` + +Rules: + +- The declaring entry is a **complete, normal entry**. It keeps its own + `files`/`overrides` and stays installable on every host and by every older + LocalAI release, which simply ignore `variants`. +- A variant references another gallery entry **by name**. That entry must exist + and must not declare `variants` of its own. +- **Order carries no meaning.** Do not try to encode a preference; write the + list in whatever order reads best. +- **Do not describe hardware.** At install time LocalAI drops variants whose + backend cannot run on the host, drops those that do not fit available memory, + and installs the largest survivor, falling back to the declaring entry's own + build. Sizes are measured live from the weights and cached, so nothing has to + be written down. +- `min_memory` on a single variant (e.g. `min_memory: 20GiB`) overrides the + measured size and suppresses the measurement for that variant. Use it only + when you have measured a real load and know the estimate is wrong; most + variants should not carry it. + +Users can override the automatic choice with `variant` on `POST /models/apply`, +`local-ai models install --variant`, or the `install_model` MCP tool. See +`docs/content/features/model-gallery.md`. + +The gallery lint specs live in `core/gallery`, so run that suite after adding a +`variants` list. + ## Available template configs Look at existing `.yaml` files in `gallery/` to find the right prompt template for your model architecture: diff --git a/core/cli/models.go b/core/cli/models.go index a95462da1864..08bbf984730f 100644 --- a/core/cli/models.go +++ b/core/cli/models.go @@ -39,6 +39,7 @@ type ModelsInstall struct { RequireBackendIntegrity bool `env:"LOCALAI_REQUIRE_BACKEND_INTEGRITY,REQUIRE_BACKEND_INTEGRITY" help:"If true, reject backend installs without a configured signature verification policy (OCI URIs) or SHA256 (tarball/HTTP URIs)." group:"hardening" default:"false"` AutoloadBackendGalleries bool `env:"LOCALAI_AUTOLOAD_BACKEND_GALLERIES" help:"If true, automatically loads backend galleries" group:"backends" default:"true"` ModelArgs []string `arg:"" optional:"" name:"models" help:"Model configuration URLs to load"` + Variant string `name:"variant" help:"Install a specific variant of a gallery entry that declares them, by the variant's model name. Leave unset to let LocalAI auto-select the largest build this machine can run." group:"models"` ModelsCMDFlags `embed:""` } @@ -147,7 +148,11 @@ func (mi *ModelsInstall) Run(ctx *cliContext.Context) error { } modelLoader := model.NewModelLoader(systemState) - err = startup.InstallModels(context.Background(), galleryService, galleries, backendGalleries, systemState, modelLoader, !mi.DisablePredownloadScan, mi.AutoloadBackendGalleries, mi.RequireBackendIntegrity, progressCallback, modelName) + var installOptions []gallery.InstallOption + if mi.Variant != "" { + installOptions = append(installOptions, gallery.WithVariant(mi.Variant)) + } + err = startup.InstallModelsWithOptions(context.Background(), galleryService, galleries, backendGalleries, systemState, modelLoader, !mi.DisablePredownloadScan, mi.AutoloadBackendGalleries, mi.RequireBackendIntegrity, progressCallback, installOptions, modelName) if err != nil { return err } diff --git a/core/gallery/describe_variants.go b/core/gallery/describe_variants.go new file mode 100644 index 000000000000..a44864e00e62 --- /dev/null +++ b/core/gallery/describe_variants.go @@ -0,0 +1,85 @@ +package gallery + +// VariantView is one selectable build of an entry, as a client sees it. +// +// It is the read-only mirror of what selection decides at install time, so a +// picker can show the user the same facts the installer acts on rather than a +// second, independently-derived opinion that could disagree with it. +type VariantView struct { + // Model is the name to send back as the install request's `variant`. + Model string `json:"model"` + // Backend is the engine the referenced entry resolves to. A client renders + // it, and it is also the reason Fits may be false on a host whose memory + // would be ample. + Backend string `json:"backend"` + // MemoryBytes is the measured footprint, or 0 when it could not be + // determined. Zero is unknown, never "needs nothing", so a client must + // render it as unknown rather than as a free option. + MemoryBytes uint64 `json:"memory_bytes,omitempty"` + // Fits reports whether auto-selection would consider this variant on this + // host: its backend can run here and its known footprint is within budget. + // An unknown footprint counts as fitting, exactly as selection treats it. + Fits bool `json:"fits"` + // IsBase marks the entry's own payload. It is always installable and is + // what auto-selection falls back to, which is why it is listed alongside + // the declared variants rather than hidden. + IsBase bool `json:"is_base"` +} + +// EntryVariants is the variant surface of a single gallery entry. +type EntryVariants struct { + Variants []VariantView `json:"variants"` + // AutoSelected is the variant auto-selection would install on this host + // right now. Clients show it as the default choice. + AutoSelected string `json:"auto_selected"` +} + +// DescribeVariants reports an entry's selectable builds and which one +// auto-selection would currently pick. +// +// It runs the SAME variantOptions + SelectVariant pass the installer runs, so +// the reported auto-selection cannot drift from what installing would actually +// do. The env carries the same probe seam too, so the size shown here and the +// size the installer compares against come from one cache and one round trip. +// +// An entry that declares no variants returns nil, nil WITHOUT touching the +// probe. That is load-bearing: the gallery listing walks entries by the +// thousand and the overwhelming majority declare nothing, so the no-variant +// case must cost nothing at all. +func DescribeVariants(models []*GalleryModel, entry *GalleryModel, env ResolveEnv) (*EntryVariants, error) { + if entry == nil || !entry.HasVariants() { + return nil, nil + } + + options, err := variantOptions(models, entry, env) + if err != nil { + return nil, err + } + + selection, err := SelectVariant(options, env, "") + if err != nil { + return nil, err + } + + views := make([]VariantView, 0, len(options)) + for _, o := range options { + memory, known, err := o.EffectiveMemory() + if err != nil { + return nil, err + } + view := VariantView{ + Model: o.Variant.Model, + Backend: o.Backend, + // The base is exempt from every filter and always installs, so + // reporting it as anything but fitting would misdescribe it. + Fits: o.IsBase || (env.backendRuns(o.Backend) && (!known || memory <= env.AvailableMemory)), + IsBase: o.IsBase, + } + if known { + view.MemoryBytes = memory + } + views = append(views, view) + } + + return &EntryVariants{Variants: views, AutoSelected: selection.Option.Variant.Model}, nil +} diff --git a/core/gallery/describe_variants_test.go b/core/gallery/describe_variants_test.go new file mode 100644 index 000000000000..4831aff15d14 --- /dev/null +++ b/core/gallery/describe_variants_test.go @@ -0,0 +1,247 @@ +package gallery_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/gallery" +) + +var _ = Describe("DescribeVariants", func() { + gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 } + + newModel := func(name, backend string) *gallery.GalleryModel { + m := &gallery.GalleryModel{} + m.Name = name + m.Backend = backend + return m + } + + var models []*gallery.GalleryModel + var base *gallery.GalleryModel + // probed records every entry the probe was asked about, so a spec can + // assert on the ABSENCE of a round trip and not merely on the result. + var probed []string + + // probing builds an env whose sizes are injected rather than measured, so + // these specs pin exact footprints without reaching the network. + probing := func(available uint64, sizes map[string]uint64) gallery.ResolveEnv { + return gallery.ResolveEnv{ + AvailableMemory: available, + BackendCompatible: func(string) bool { return true }, + ProbeMemory: func(target *gallery.GalleryModel) uint64 { + probed = append(probed, target.Name) + return sizes[target.Name] + }, + } + } + + byName := func(view *gallery.EntryVariants, name string) gallery.VariantView { + GinkgoHelper() + for _, v := range view.Variants { + if v.Model == name { + return v + } + } + Fail("no variant named " + name + " in the reported view") + return gallery.VariantView{} + } + + BeforeEach(func() { + probed = nil + big := newModel("qwen3-8b-vllm-awq", "vllm") + small := newModel("qwen3-8b-gguf-q8", "llama-cpp") + base = newModel("qwen3-8b-gguf-q4", "llama-cpp") + base.Variants = []gallery.Variant{ + {Model: "qwen3-8b-vllm-awq"}, + {Model: "qwen3-8b-gguf-q8"}, + } + models = []*gallery.GalleryModel{big, small, base} + }) + + Describe("an entry that declares no variants", func() { + It("reports nothing and issues no probe at all", func() { + // This is the performance contract of the gallery listing: the + // listing walks over a thousand entries and virtually none of them + // declare variants, so the ordinary entry must cost zero round + // trips. Asserting on `probed` rather than on timing makes that + // contract testable. + plain := newModel("plain-entry", "llama-cpp") + models = append(models, plain) + + view, err := gallery.DescribeVariants(models, plain, probing(gib(24), map[string]uint64{ + "plain-entry": gib(4), + })) + + Expect(err).ToNot(HaveOccurred()) + Expect(view).To(BeNil()) + Expect(probed).To(BeEmpty()) + }) + + It("reports nothing for a nil entry rather than panicking", func() { + view, err := gallery.DescribeVariants(models, nil, probing(gib(24), nil)) + Expect(err).ToNot(HaveOccurred()) + Expect(view).To(BeNil()) + Expect(probed).To(BeEmpty()) + }) + }) + + Describe("an entry that declares variants", func() { + It("reports every declared variant plus the entry's own build", func() { + view, err := gallery.DescribeVariants(models, base, probing(gib(24), map[string]uint64{ + "qwen3-8b-vllm-awq": gib(20), + "qwen3-8b-gguf-q8": gib(9), + })) + Expect(err).ToNot(HaveOccurred()) + + names := []string{} + for _, v := range view.Variants { + names = append(names, v.Model) + } + // The base is listed so a picker can offer "decline the upgrade", + // which is a real choice an operator makes. + Expect(names).To(ConsistOf("qwen3-8b-vllm-awq", "qwen3-8b-gguf-q8", "qwen3-8b-gguf-q4")) + Expect(byName(view, "qwen3-8b-gguf-q4").IsBase).To(BeTrue()) + Expect(byName(view, "qwen3-8b-vllm-awq").IsBase).To(BeFalse()) + }) + + It("reports the backend of the referenced entry, not of the declaring one", func() { + // A picker renders this, and it is also the reason a variant can be + // unavailable on a host with ample memory. + view, err := gallery.DescribeVariants(models, base, probing(gib(24), nil)) + Expect(err).ToNot(HaveOccurred()) + Expect(byName(view, "qwen3-8b-vllm-awq").Backend).To(Equal("vllm")) + Expect(byName(view, "qwen3-8b-gguf-q8").Backend).To(Equal("llama-cpp")) + }) + + It("reports the probed size of each variant", func() { + view, err := gallery.DescribeVariants(models, base, probing(gib(24), map[string]uint64{ + "qwen3-8b-vllm-awq": gib(20), + "qwen3-8b-gguf-q8": gib(9), + })) + Expect(err).ToNot(HaveOccurred()) + Expect(byName(view, "qwen3-8b-vllm-awq").MemoryBytes).To(Equal(gib(20))) + Expect(byName(view, "qwen3-8b-gguf-q8").MemoryBytes).To(Equal(gib(9))) + }) + + It("lets an authored min_memory win over the probe and skips that probe", func() { + base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq", MinMemory: "20GiB"}} + + view, err := gallery.DescribeVariants(models, base, probing(gib(24), map[string]uint64{ + "qwen3-8b-vllm-awq": gib(2), + })) + Expect(err).ToNot(HaveOccurred()) + Expect(byName(view, "qwen3-8b-vllm-awq").MemoryBytes).To(Equal(gib(20))) + Expect(probed).ToNot(ContainElement("qwen3-8b-vllm-awq")) + }) + + It("reports a size it could not determine as unknown rather than as free", func() { + // Zero on the wire means unknown. Rendering it as a zero-byte model + // would advertise the largest download on offer as costless. + view, err := gallery.DescribeVariants(models, base, probing(gib(24), map[string]uint64{})) + Expect(err).ToNot(HaveOccurred()) + Expect(byName(view, "qwen3-8b-vllm-awq").MemoryBytes).To(Equal(uint64(0))) + Expect(byName(view, "qwen3-8b-vllm-awq").Fits).To(BeTrue()) + }) + + It("marks a variant too large for this host as not fitting", func() { + view, err := gallery.DescribeVariants(models, base, probing(gib(10), map[string]uint64{ + "qwen3-8b-vllm-awq": gib(20), + "qwen3-8b-gguf-q8": gib(9), + })) + Expect(err).ToNot(HaveOccurred()) + Expect(byName(view, "qwen3-8b-vllm-awq").Fits).To(BeFalse()) + Expect(byName(view, "qwen3-8b-gguf-q8").Fits).To(BeTrue()) + }) + + It("marks a variant whose backend cannot run here as not fitting, however much memory there is", func() { + env := probing(gib(1024), map[string]uint64{ + "qwen3-8b-vllm-awq": gib(20), + "qwen3-8b-gguf-q8": gib(9), + }) + env.BackendCompatible = func(backend string) bool { return backend != "vllm" } + + view, err := gallery.DescribeVariants(models, base, env) + Expect(err).ToNot(HaveOccurred()) + Expect(byName(view, "qwen3-8b-vllm-awq").Fits).To(BeFalse()) + Expect(byName(view, "qwen3-8b-gguf-q8").Fits).To(BeTrue()) + }) + + It("always reports the entry's own build as fitting", func() { + // The base is exempt from every filter and always installs, so + // showing it as unavailable would misdescribe it. Neither a hostile + // backend gate nor a zero memory budget may change that. + env := probing(0, map[string]uint64{}) + env.BackendCompatible = func(string) bool { return false } + + view, err := gallery.DescribeVariants(models, base, env) + Expect(err).ToNot(HaveOccurred()) + Expect(byName(view, "qwen3-8b-gguf-q4").Fits).To(BeTrue()) + }) + + It("surfaces a variant that references an entry no gallery declares", func() { + base.Variants = []gallery.Variant{{Model: "does-not-exist"}} + _, err := gallery.DescribeVariants(models, base, probing(gib(24), nil)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does-not-exist")) + }) + }) + + Describe("the reported auto-selection", func() { + // These are the specs that keep a picker honest: what the listing shows + // as the default must be what installing with no variant actually does. + // They assert against ResolveVariant rather than against a hardcoded + // name, so a change to the selection rules cannot make the two drift + // without failing here. + agreesWithInstall := func(env gallery.ResolveEnv) { + GinkgoHelper() + view, err := gallery.DescribeVariants(models, base, env) + Expect(err).ToNot(HaveOccurred()) + + _, chosen, err := gallery.ResolveVariant(models, base, env, "") + Expect(err).ToNot(HaveOccurred()) + Expect(view.AutoSelected).To(Equal(chosen.Model)) + } + + It("matches what installing would pick when everything fits", func() { + agreesWithInstall(probing(gib(64), map[string]uint64{ + "qwen3-8b-vllm-awq": gib(20), + "qwen3-8b-gguf-q8": gib(9), + })) + }) + + It("matches what installing would pick when only the smaller variant fits", func() { + agreesWithInstall(probing(gib(12), map[string]uint64{ + "qwen3-8b-vllm-awq": gib(20), + "qwen3-8b-gguf-q8": gib(9), + })) + }) + + It("matches what installing would pick when nothing fits and the base wins", func() { + agreesWithInstall(probing(gib(2), map[string]uint64{ + "qwen3-8b-vllm-awq": gib(20), + "qwen3-8b-gguf-q8": gib(9), + })) + }) + + It("names the largest fitting variant, not the first authored", func() { + // Pinned separately from agreesWithInstall so that a mutation making + // BOTH functions pick the wrong variant still fails a spec. + view, err := gallery.DescribeVariants(models, base, probing(gib(64), map[string]uint64{ + "qwen3-8b-vllm-awq": gib(20), + "qwen3-8b-gguf-q8": gib(9), + })) + Expect(err).ToNot(HaveOccurred()) + Expect(view.AutoSelected).To(Equal("qwen3-8b-vllm-awq")) + }) + + It("names the entry itself when no variant fits", func() { + view, err := gallery.DescribeVariants(models, base, probing(gib(2), map[string]uint64{ + "qwen3-8b-vllm-awq": gib(20), + "qwen3-8b-gguf-q8": gib(9), + })) + Expect(err).ToNot(HaveOccurred()) + Expect(view.AutoSelected).To(Equal("qwen3-8b-gguf-q4")) + }) + }) +}) diff --git a/core/gallery/models.go b/core/gallery/models.go index 1792d415d4ee..9cc492125700 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -252,6 +252,31 @@ func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, return &resolved, selected.Variant, nil } +// HostResolveEnv describes this machine to variant selection. +// +// It exists so the install path and every read-only surface that reports what +// selection WOULD choose derive the host from one place. Two copies of this +// wiring would eventually disagree, and a picker that shows a different answer +// than the installer produces is worse than no picker at all. +func HostResolveEnv(ctx context.Context, systemState *system.SystemState) ResolveEnv { + return ResolveEnv{ + AvailableMemory: availableModelMemory(systemState), + // The whole hardware gate. IsBackendCompatible already derives + // Darwin-only, NVIDIA-only, ROCm-only and SYCL-only from the backend + // name, so a gallery author never has to describe hardware. + // + // The uri argument is deliberately empty: it exists for backend OCI + // images, and passing a model's gallery url here would let an unrelated + // substring in a download link decide hardware compatibility. + BackendCompatible: func(backend string) bool { + return systemState.IsBackendCompatible(backend, "") + }, + ProbeMemory: func(target *GalleryModel) uint64 { + return probeEntryMemory(ctx, target) + }, + } +} + // noGPUDetected is what SystemState.DetectedCapability() reports when no // usable GPU was found. The constant is unexported in pkg/system. const noGPUDetected = "default" @@ -419,6 +444,12 @@ func InstallModelFromGallery( // still installable as-is; selection below only decides whether one of its // declared alternatives suits this host better. if !model.HasVariants() { + // A caller who named a variant asked for something this entry cannot + // give. Installing the entry anyway would look like success while + // quietly ignoring the choice, so it is refused by name instead. + if installOpts.variant != "" { + return fmt.Errorf("%w: %q was requested but model %q declares no variants", ErrPinNotFound, installOpts.variant, model.Name) + } return applyModel(model, nil) } @@ -443,22 +474,7 @@ func InstallModelFromGallery( } } - env := ResolveEnv{ - AvailableMemory: availableModelMemory(systemState), - // The whole hardware gate. IsBackendCompatible already derives - // Darwin-only, NVIDIA-only, ROCm-only and SYCL-only from the backend - // name, so a gallery author never has to describe hardware. - // - // The uri argument is deliberately empty: it exists for backend OCI - // images, and passing a model's gallery url here would let an unrelated - // substring in a download link decide hardware compatibility. - BackendCompatible: func(backend string) bool { - return systemState.IsBackendCompatible(backend, "") - }, - ProbeMemory: func(target *GalleryModel) uint64 { - return probeEntryMemory(ctx, target) - }, - } + env := HostResolveEnv(ctx, systemState) resolved, variant, err := ResolveVariant(models, model, env, pin) if err != nil { diff --git a/core/gallery/variants_install_test.go b/core/gallery/variants_install_test.go index 424bbfd81f7f..8bf275f33ae8 100644 --- a/core/gallery/variants_install_test.go +++ b/core/gallery/variants_install_test.go @@ -505,6 +505,36 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { Expect(installedBackend("qwen3-8b-q4")).To(Equal("upgrade-backend")) }) + It("refuses a variant name the entry does not declare, naming what was asked for", func() { + newGallery( + withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}), + entry("qwen3-8b-q8", "upgrade-backend"), + ) + + err := install("qwen3-8b-q4", gallery.GalleryModel{}, gallery.WithVariant("qwen3-8b-q6")) + + // Auto-selecting instead would report success while silently + // installing something other than what the caller asked for, and a + // typo'd variant name would be indistinguishable from a working one. + Expect(err).To(MatchError(gallery.ErrPinNotFound)) + Expect(err.Error()).To(ContainSubstring("qwen3-8b-q6")) + Expect(filepath.Join(tempdir, "qwen3-8b-q4.yaml")).ToNot(BeAnExistingFile()) + }) + + It("refuses a variant name when the entry declares no variants at all", func() { + // The entry is installable and selection never runs for it, so without + // an explicit refusal the requested variant would be dropped on the + // floor and the install would look like it honored the choice. + newGallery(entry("qwen3-8b-q4", "base-backend")) + + err := install("qwen3-8b-q4", gallery.GalleryModel{}, gallery.WithVariant("qwen3-8b-q8")) + + Expect(err).To(MatchError(gallery.ErrPinNotFound)) + Expect(err.Error()).To(ContainSubstring("qwen3-8b-q8")) + Expect(filepath.Join(tempdir, "qwen3-8b-q4.yaml")).ToNot(BeAnExistingFile()) + }) + It("records a pin and honors it on a plain reinstall", func() { newGallery( withVariants(entry("qwen3-8b-q4", "base-backend"), diff --git a/core/http/endpoints/localai/gallery.go b/core/http/endpoints/localai/gallery.go index fabae336d786..fb0a77bb4e1d 100644 --- a/core/http/endpoints/localai/gallery.go +++ b/core/http/endpoints/localai/gallery.go @@ -25,6 +25,11 @@ type ModelGalleryEndpointService struct { type GalleryModel struct { ID string `json:"id"` + // Variant installs one specific build of an entry that declares variants, + // named as it appears in the entry's `variants` list (see the `variants` + // and `auto_variant` fields of the gallery listing). Leave it empty to let + // LocalAI auto-select the largest build this host can actually run. + Variant string `json:"variant,omitempty"` gallery.GalleryModel } @@ -86,6 +91,7 @@ func (mgs *ModelGalleryEndpointService) ApplyModelGalleryEndpoint() echo.Handler Req: input.GalleryModel, ID: uuid.String(), GalleryElementName: input.ID, + Variant: input.Variant, Galleries: mgs.galleries, BackendGalleries: mgs.backendGalleries, } diff --git a/core/http/routes/ui_api.go b/core/http/routes/ui_api.go index e737e445e00b..b30dd256179e 100644 --- a/core/http/routes/ui_api.go +++ b/core/http/routes/ui_api.go @@ -413,6 +413,11 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model }) } + // Kept before the filters and pagination below narrow `models`: a + // variant references another gallery entry by name, and that entry may + // well have been filtered off this page. + allModels := models + // Get all available tags allTags := map[string]struct{}{} tags := []string{} @@ -585,6 +590,26 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model "voice_cloning": m.VoiceCloningCapability(appConfig.SystemState.Model.ModelsPath), } + // Variants are described only for the entries that declare them. + // DescribeVariants is what probes model sizes, so calling it for + // every entry would put a network round trip behind each of the + // ~1200 entries in the gallery. The HasVariants guard keeps an + // ordinary entry exactly as cheap as it was before variants + // existed; only a declaring entry pays, and only on the page it + // appears on. + if m.HasVariants() { + env := gallery.HostResolveEnv(c.Request().Context(), appConfig.SystemState) + view, describeErr := gallery.DescribeVariants(allModels, m, env) + if describeErr != nil { + // A malformed variant list must not blank the whole gallery + // page; that entry just renders without a picker. + xlog.Warn("could not describe model variants", "model", m.Name, "error", describeErr) + } else if view != nil { + obj["variants"] = view.Variants + obj["auto_variant"] = view.AutoSelected + } + } + modelsJSON = append(modelsJSON, obj) } @@ -809,7 +834,11 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model "error": "invalid model ID", }) } - xlog.Debug("API job submitted to install", "galleryID", galleryID) + // Optional: one of the variants the listing reported for this entry. + // Absent means auto-select, which is what the listing's auto_variant + // already told the client would happen. + variant := c.QueryParam("variant") + xlog.Debug("API job submitted to install", "galleryID", galleryID, "variant", variant) id, err := uuid.NewUUID() if err != nil { @@ -825,6 +854,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model op := galleryop.ManagementOp[gallery.GalleryModel, gallery.ModelConfig]{ ID: uid, GalleryElementName: galleryID, + Variant: variant, Galleries: appConfig.Galleries, BackendGalleries: appConfig.BackendGalleries, Context: ctx, diff --git a/core/services/galleryop/managers_local.go b/core/services/galleryop/managers_local.go index 10fb29491b5b..70fc018e000b 100644 --- a/core/services/galleryop/managers_local.go +++ b/core/services/galleryop/managers_local.go @@ -62,10 +62,14 @@ func (m *LocalModelManager) InstallModel(ctx context.Context, op *ManagementOp[g } return nil case op.GalleryElementName != "": + opts := []gallery.InstallOption{gallery.WithArtifactMaterializer(m.artifactMaterializer)} + if op.Variant != "" { + opts = append(opts, gallery.WithVariant(op.Variant)) + } return gallery.InstallModelFromGallery(ctx, op.Galleries, op.BackendGalleries, m.systemState, m.modelLoader, op.GalleryElementName, op.Req, progressCb, m.enforcePredownloadScans, m.automaticallyInstallBackend, m.requireBackendIntegrity, - gallery.WithArtifactMaterializer(m.artifactMaterializer)) + opts...) default: return installModelFromRemoteConfig(ctx, m.systemState, m.modelLoader, op.Req, progressCb, m.enforcePredownloadScans, m.automaticallyInstallBackend, op.BackendGalleries, m.requireBackendIntegrity, diff --git a/core/services/galleryop/model_variant_test.go b/core/services/galleryop/model_variant_test.go new file mode 100644 index 000000000000..ada429130b3c --- /dev/null +++ b/core/services/galleryop/model_variant_test.go @@ -0,0 +1,102 @@ +package galleryop_test + +import ( + "context" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/services/galleryop" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +// A variant chosen in the UI or over REST arrives here as ManagementOp.Variant. +// If it is not turned into a gallery.WithVariant install option the install +// still succeeds, just on the auto-selected build, so the user's choice +// disappears without a single error anywhere. These specs pin the threading. +var _ = Describe("LocalModelManager variant selection", func() { + var ( + modelsDir string + mgr *galleryop.LocalModelManager + appConfig *config.ApplicationConfig + systemState *system.SystemState + ) + + // Every entry carries an inline config_file, so the whole install runs off + // the local filesystem and never reaches the network. + entry := func(name, backend string, variants ...gallery.Variant) gallery.GalleryModel { + m := gallery.GalleryModel{ConfigFile: map[string]any{"backend": backend}} + m.Name = name + m.Description = "entry " + name + m.Variants = variants + return m + } + + installedBackend := func(name string) string { + GinkgoHelper() + dat, err := os.ReadFile(filepath.Join(modelsDir, name+".yaml")) + Expect(err).ToNot(HaveOccurred()) + content := map[string]any{} + Expect(yaml.Unmarshal(dat, &content)).To(Succeed()) + return content["backend"].(string) + } + + install := func(variant string) error { + op := &galleryop.ManagementOp[gallery.GalleryModel, gallery.ModelConfig]{ + GalleryElementName: "qwen3-8b-q4", + Variant: variant, + Galleries: appConfig.Galleries, + } + return mgr.InstallModel(context.Background(), op, func(string, string, string, float64) {}) + } + + BeforeEach(func() { + var err error + modelsDir, err = os.MkdirTemp("", "variant-op-*") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { Expect(os.RemoveAll(modelsDir)).To(Succeed()) }) + + // "0GiB" always fits, so auto-selection would take the upgrade on any + // machine. Only an honored pin can land the install on the base. + entries := []gallery.GalleryModel{ + entry("qwen3-8b-q4", "base-backend", gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}), + entry("qwen3-8b-q8", "upgrade-backend"), + } + out, err := yaml.Marshal(entries) + Expect(err).ToNot(HaveOccurred()) + galleryPath := filepath.Join(modelsDir, "gallery.yaml") + Expect(os.WriteFile(galleryPath, out, 0o600)).To(Succeed()) + + systemState, err = system.GetSystemState(system.WithModelPath(modelsDir)) + Expect(err).ToNot(HaveOccurred()) + appConfig = &config.ApplicationConfig{ + SystemState: systemState, + Galleries: []config.Gallery{{Name: "test", URL: "file://" + galleryPath}}, + } + mgr = galleryop.NewLocalModelManager(appConfig, model.NewModelLoader(systemState)) + }) + + It("installs the entry's own build when the op pins it", func() { + Expect(install("qwen3-8b-q4")).To(Succeed()) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("base-backend")) + }) + + It("auto-selects when the op names no variant", func() { + // The mirror of the spec above: same gallery, same host, only the + // pin differs, so nothing but the pin can explain the different build. + Expect(install("")).To(Succeed()) + Expect(installedBackend("qwen3-8b-q4")).To(Equal("upgrade-backend")) + }) + + It("fails the install when the op names a variant the entry does not declare", func() { + err := install("qwen3-8b-q2") + Expect(err).To(MatchError(gallery.ErrPinNotFound)) + Expect(err.Error()).To(ContainSubstring("qwen3-8b-q2")) + }) +}) diff --git a/core/services/galleryop/models.go b/core/services/galleryop/models.go index 1c6e57c2a918..e6a965b28565 100644 --- a/core/services/galleryop/models.go +++ b/core/services/galleryop/models.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "os" + "slices" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" @@ -174,6 +175,9 @@ func installModelFromRemoteConfig(ctx context.Context, systemState *system.Syste type galleryModel struct { gallery.GalleryModel `yaml:",inline"` // https://github.com/go-yaml/yaml/issues/63 ID string `json:"id"` + // Variant pins the install to one of the entry's declared variants. Empty + // means auto-select. + Variant string `json:"variant,omitempty" yaml:"variant,omitempty"` } func processRequests(systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, automaticallyInstallBackend bool, galleries []config.Gallery, backendGalleries []config.Gallery, requests []galleryModel, requireBackendIntegrity bool, options ...gallery.InstallOption) error { @@ -185,8 +189,15 @@ func processRequests(systemState *system.SystemState, modelLoader *model.ModelLo err = installModelFromRemoteConfig(ctx, systemState, modelLoader, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, backendGalleries, requireBackendIntegrity, options...) } else { + // Cloned rather than appended to in place: `options` is shared by + // every request in the batch, so appending would leak one request's + // pin onto the next request that reuses the same backing array. + requestOptions := options + if r.Variant != "" { + requestOptions = append(slices.Clone(options), gallery.WithVariant(r.Variant)) + } err = gallery.InstallModelFromGallery( - ctx, galleries, backendGalleries, systemState, modelLoader, r.ID, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, requireBackendIntegrity, options...) + ctx, galleries, backendGalleries, systemState, modelLoader, r.ID, r.GalleryModel, utils.DisplayDownloadFunction, enforceScan, automaticallyInstallBackend, requireBackendIntegrity, requestOptions...) } } return err diff --git a/core/services/galleryop/operation.go b/core/services/galleryop/operation.go index ed86f75a840a..7eb7cedbc4ab 100644 --- a/core/services/galleryop/operation.go +++ b/core/services/galleryop/operation.go @@ -43,6 +43,15 @@ type ManagementOp[T any, E any] struct { // build on one node without touching the rest of the cluster. TargetNodeID string + // Variant pins a model install to one of the gallery entry's declared + // variants, by that variant's model name. Empty means auto-select: LocalAI + // picks the largest variant this host's backend support and memory can + // actually run, and falls back to the entry's own build. + // + // A name that is not among the entry's variants fails the install rather + // than quietly auto-selecting, so a typo cannot masquerade as a choice. + Variant string + // Upgrade is true if this is an upgrade operation (not a fresh install) Upgrade bool diff --git a/core/startup/model_preload.go b/core/startup/model_preload.go index 3ca375b04b8e..4f3bb16832d5 100644 --- a/core/startup/model_preload.go +++ b/core/startup/model_preload.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "time" "github.com/google/uuid" @@ -22,9 +23,17 @@ import ( // It will download the model if it is not already present in the model path // It will also try to resolve if the model is an embedded model YAML configuration func InstallModels(ctx context.Context, galleryService *galleryop.GalleryService, galleries, backendGalleries []config.Gallery, systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, autoloadBackendGalleries, requireBackendIntegrity bool, downloadStatus func(string, string, string, float64), models ...string) error { + return InstallModelsWithOptions(ctx, galleryService, galleries, backendGalleries, systemState, modelLoader, enforceScan, autoloadBackendGalleries, requireBackendIntegrity, downloadStatus, nil, models...) +} + +// InstallModelsWithOptions is InstallModels with extra install options, which +// is how the CLI passes a variant pin. It is a separate entry point rather than +// a variadic tail on InstallModels because that signature already ends in a +// variadic model list. +func InstallModelsWithOptions(ctx context.Context, galleryService *galleryop.GalleryService, galleries, backendGalleries []config.Gallery, systemState *system.SystemState, modelLoader *model.ModelLoader, enforceScan, autoloadBackendGalleries, requireBackendIntegrity bool, downloadStatus func(string, string, string, float64), extraOptions []gallery.InstallOption, models ...string) error { // create an error that groups all errors var err error - var installOptions []gallery.InstallOption + installOptions := slices.Clone(extraOptions) if galleryService != nil { installOptions = append(installOptions, gallery.WithArtifactMaterializer(galleryService.ModelArtifactMaterializer())) } diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index 29f72e682c96..68b08a0cf08d 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -151,6 +151,70 @@ where: - `bert-embeddings` is the model name in the gallery (read its [config here](https://github.com/mudler/LocalAI/tree/master/gallery/blob/main/bert-embeddings.yaml)). +### Model variants + +Some gallery entries offer several builds of the same model: different +quantizations, or the same weights served by a different engine. Such an entry +carries a `variants` list, and installing it normally lets LocalAI choose: + +- variants whose backend cannot run on this machine are dropped; +- variants that do not fit the available memory (VRAM on a GPU host, otherwise + system RAM) are dropped; +- the largest remaining variant wins, because a bigger footprint means a higher + quality build of the same model; +- if nothing survives, the entry's own build is installed. The entry is always + installable, on any machine. + +Sizes are measured from the model's weights rather than downloaded, and cached. + +The gallery listing reports what an entry offers. Entries with variants carry +two extra fields, `variants` and `auto_variant`: + +```bash +curl http://localhost:8080/api/models | jq '.models[] | select(.variants) | + {name, auto_variant, variants}' +``` + +```json +{ + "name": "nanbeige4.1-3b-q4", + "auto_variant": "nanbeige4.1-3b-q8", + "variants": [ + { "model": "nanbeige4.1-3b-q8", "backend": "llama-cpp", "memory_bytes": 4187593113, "fits": true, "is_base": false }, + { "model": "nanbeige4.1-3b-q4", "backend": "llama-cpp", "memory_bytes": 0, "fits": true, "is_base": true } + ] +} +``` + +`auto_variant` is what installing without a choice would pick right now. `fits` +is whether auto-selection would consider that variant on this machine, and a +`memory_bytes` of 0 means the size could not be determined, not that the build +is free. `is_base` marks the entry's own build. + +To install a specific one, pass its name as `variant`: + +```bash +curl $LOCALAI/models/apply -H "Content-Type: application/json" -d '{ + "id": "localai@nanbeige4.1-3b-q4", + "variant": "nanbeige4.1-3b-q8" + }' +``` + +An explicit choice is honored even when the machine looks too small for it, so +you can deliberately install a build LocalAI would not have picked. A `variant` +the entry does not declare fails the install and names what was requested; it +never quietly falls back to auto-selection. The choice is recorded, so a later +reinstall or upgrade of the same model stays on the variant you picked. + +The same option exists on the CLI: + +```bash +local-ai models install nanbeige4.1-3b-q4 --variant nanbeige4.1-3b-q8 +``` + +Entries without a `variants` list are unaffected by any of this and install +exactly as they always have. + ### Artifact-backed models Gallery models with an `artifacts` declaration are fully materialized during diff --git a/pkg/mcp/localaitools/dto.go b/pkg/mcp/localaitools/dto.go index ecc23204a734..ab354b31d08b 100644 --- a/pkg/mcp/localaitools/dto.go +++ b/pkg/mcp/localaitools/dto.go @@ -67,6 +67,7 @@ type InstallModelRequest struct { GalleryName string `json:"gallery_name,omitempty" jsonschema:"The gallery the model lives in (from gallery_search). Optional when ModelName is unique across galleries."` ModelName string `json:"model_name" jsonschema:"The canonical model name as returned by gallery_search."` Overrides map[string]any `json:"overrides,omitempty" jsonschema:"Optional config overrides to merge into the installed model's YAML."` + Variant string `json:"variant,omitempty" jsonschema:"Optional. Installs one specific build of an entry that offers several (different quantizations or engines), named exactly as it appears in that entry's variant list. Leave empty unless the user explicitly asked for a particular build: by default LocalAI auto-selects the largest variant this machine's engine support and free memory can actually run."` } // InstallBackendRequest is the input for install_backend. diff --git a/pkg/mcp/localaitools/httpapi/client.go b/pkg/mcp/localaitools/httpapi/client.go index 771d2908a870..8029655caa29 100644 --- a/pkg/mcp/localaitools/httpapi/client.go +++ b/pkg/mcp/localaitools/httpapi/client.go @@ -252,6 +252,9 @@ func (c *Client) InstallModel(ctx context.Context, req localaitools.InstallModel if len(req.Overrides) > 0 { body["overrides"] = req.Overrides } + if req.Variant != "" { + body["variant"] = req.Variant + } var resp struct { ID string `json:"uuid"` StatusURL string `json:"status"` diff --git a/pkg/mcp/localaitools/httpapi/client_test.go b/pkg/mcp/localaitools/httpapi/client_test.go index 319ceffee50b..38eabf2832b7 100644 --- a/pkg/mcp/localaitools/httpapi/client_test.go +++ b/pkg/mcp/localaitools/httpapi/client_test.go @@ -129,6 +129,39 @@ var _ = Describe("httpapi.Client against the LocalAI admin REST surface", func() Expect(err).ToNot(HaveOccurred()) Expect(id).To(Equal("job-123")) }) + + It("forwards a chosen variant to the apply endpoint", func() { + // The assistant's whole ability to honor "install the Q8 one" + // rests on this field surviving the hop to REST. + var body map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(json.NewDecoder(r.Body).Decode(&body)).To(Succeed()) + _ = json.NewEncoder(w).Encode(map[string]any{"uuid": "job-123"}) + })) + DeferCleanup(srv.Close) + + _, err := New(srv.URL, "").InstallModel(context.Background(), localaitools.InstallModelRequest{ + ModelName: "qwen2.5-7b-instruct", + Variant: "qwen2.5-7b-instruct-q8", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(body).To(HaveKeyWithValue("variant", "qwen2.5-7b-instruct-q8")) + }) + + It("omits the variant when none was chosen, so the server auto-selects", func() { + var body map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(json.NewDecoder(r.Body).Decode(&body)).To(Succeed()) + _ = json.NewEncoder(w).Encode(map[string]any{"uuid": "job-123"}) + })) + DeferCleanup(srv.Close) + + _, err := New(srv.URL, "").InstallModel(context.Background(), localaitools.InstallModelRequest{ + ModelName: "qwen2.5-7b-instruct", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(body).ToNot(HaveKey("variant")) + }) }) Describe("GetJobStatus", func() { diff --git a/pkg/mcp/localaitools/inproc/client.go b/pkg/mcp/localaitools/inproc/client.go index 9f6eb8fe20f6..d3054f614ba0 100644 --- a/pkg/mcp/localaitools/inproc/client.go +++ b/pkg/mcp/localaitools/inproc/client.go @@ -203,6 +203,7 @@ func (c *Client) InstallModel(ctx context.Context, req localaitools.InstallModel Req: gallery.GalleryModel{ Metadata: gallery.Metadata{Name: req.ModelName}, }, + Variant: req.Variant, Galleries: galleries, BackendGalleries: c.AppConfig.BackendGalleries, } diff --git a/pkg/mcp/localaitools/prompts/skills/install_chat_model.md b/pkg/mcp/localaitools/prompts/skills/install_chat_model.md index ca6bd12e2352..54d2569d1cef 100644 --- a/pkg/mcp/localaitools/prompts/skills/install_chat_model.md +++ b/pkg/mcp/localaitools/prompts/skills/install_chat_model.md @@ -6,7 +6,7 @@ Use this when the user wants to install a chat-capable model — including the c 2. Show the top results as a numbered list with name, gallery, short description, and license. If none match, say so and ask whether to broaden the search. 3. Wait for the user to pick. 4. Summarise the chosen install ("I'll install **`/`** — confirm?") and wait for confirmation. -5. On confirmation, call `install_model` with `gallery_name` and `model_name` from the chosen hit. +5. On confirmation, call `install_model` with `gallery_name` and `model_name` from the chosen hit. Leave `variant` empty: LocalAI then auto-selects the largest build this machine's engines and free memory can actually run. Only set `variant` when the user names a specific build (e.g. "the Q8 one"), and pass the name exactly as the entry lists it — an unknown name fails the install rather than falling back to auto-selection. 6. Poll `get_job_status` with the returned job id. Report meaningful progress changes (every ~10–20%, plus completion). 7. When the job reports `processed: true` and no error, call `reload_models`, then `list_installed_models` with `capability: "chat"` to confirm the model is now visible. 8. Tell the user the model is ready and how to use it (its name as the `model` field in chat completions). diff --git a/pkg/mcp/localaitools/tools_models.go b/pkg/mcp/localaitools/tools_models.go index d652198a4b76..f38e6edcef96 100644 --- a/pkg/mcp/localaitools/tools_models.go +++ b/pkg/mcp/localaitools/tools_models.go @@ -83,7 +83,7 @@ func registerModelTools(s *mcp.Server, client LocalAIClient, opts Options) { mcp.AddTool(s, &mcp.Tool{ Name: ToolInstallModel, - Description: "Install a model from a gallery. Requires explicit user confirmation per safety rule 1. Returns a job id; poll with get_job_status.", + Description: "Install a model from a gallery. Some entries offer several variants (quantizations or engines) of the same model; leaving `variant` empty auto-selects the largest one this machine can actually run, which is almost always what the user wants. Set `variant` only when the user names a specific build. Requires explicit user confirmation per safety rule 1. Returns a job id; poll with get_job_status.", }, func(ctx context.Context, _ *mcp.CallToolRequest, args InstallModelRequest) (*mcp.CallToolResult, any, error) { // Empty-string check at the tool layer: the SDK schema validator // only enforces presence, not non-empty, and we want a consistent From 84de7a68a8cb0f01f8f6e230fdafcbfc42a8bfdf Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 23:18:12 +0000 Subject: [PATCH 19/48] feat(gallery): drop the redundant variant min_memory field Variant.MinMemory was an authored override for when the live probe misreads a variant's footprint. It duplicated an existing field: probeEntryMemory already passes the entry's declared size: into EstimateModelMultiContext, whose cascade prefers that declared size over its own guesswork. Correcting size: on the referenced entry fixes the figure for every consumer rather than only for variant selection, so min_memory shadowed the right answer. A variant is now nothing but a name. Its effective size is exactly the probe result, and an unknown stays unknown: it survives the filter and ranks last. EffectiveMemory loses its error return along with the field. The authored string was the only thing that could fail to parse, so the error had no remaining source and was propagating dead nil-checks through SelectVariant, DescribeVariants and the pin warning. Selection behaviour is unchanged. The specs covering probe-derived sizing, ranking, filtering, the unknown-size path, pin recall, entry/variant metadata split and deep-copy isolation all survive; the three install specs that needed a definite size now declare it through the referenced entry's own size:, which exercises the documented escape hatch directly. gallery/index.yaml is untouched: no entry ever carried the key. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- .agents/adding-gallery-models.md | 9 +- core/gallery/describe_variants.go | 5 +- core/gallery/describe_variants_test.go | 11 -- core/gallery/models.go | 6 +- core/gallery/resolve_variant.go | 32 ++--- core/gallery/resolve_variant_test.go | 126 +++++++----------- core/gallery/variant.go | 38 +----- core/gallery/variants_install_test.go | 91 ++++++------- core/gallery/variants_lint_test.go | 73 +--------- core/schema/gallery-model.schema.json | 4 - core/services/galleryop/model_variant_test.go | 8 +- 11 files changed, 126 insertions(+), 277 deletions(-) diff --git a/.agents/adding-gallery-models.md b/.agents/adding-gallery-models.md index 9e44b0a118f3..93563c76279d 100644 --- a/.agents/adding-gallery-models.md +++ b/.agents/adding-gallery-models.md @@ -119,10 +119,11 @@ Rules: and installs the largest survivor, falling back to the declaring entry's own build. Sizes are measured live from the weights and cached, so nothing has to be written down. -- `min_memory` on a single variant (e.g. `min_memory: 20GiB`) overrides the - measured size and suppresses the measurement for that variant. Use it only - when you have measured a real load and know the estimate is wrong; most - variants should not carry it. +- A variant is nothing but a name; there is no per-variant memory field. When + the measured size for a build is wrong, correct it on the referenced entry by + setting that entry's own `size:` (e.g. `size: "20GiB"`). The estimator prefers + a declared size over its own guesswork, so the fix applies everywhere the size + is shown or compared rather than only to variant selection. Users can override the automatic choice with `variant` on `POST /models/apply`, `local-ai models install --variant`, or the `install_model` MCP tool. See diff --git a/core/gallery/describe_variants.go b/core/gallery/describe_variants.go index a44864e00e62..726308909bab 100644 --- a/core/gallery/describe_variants.go +++ b/core/gallery/describe_variants.go @@ -63,10 +63,7 @@ func DescribeVariants(models []*GalleryModel, entry *GalleryModel, env ResolveEn views := make([]VariantView, 0, len(options)) for _, o := range options { - memory, known, err := o.EffectiveMemory() - if err != nil { - return nil, err - } + memory, known := o.EffectiveMemory() view := VariantView{ Model: o.Variant.Model, Backend: o.Backend, diff --git a/core/gallery/describe_variants_test.go b/core/gallery/describe_variants_test.go index 4831aff15d14..0dbe35f39639 100644 --- a/core/gallery/describe_variants_test.go +++ b/core/gallery/describe_variants_test.go @@ -124,17 +124,6 @@ var _ = Describe("DescribeVariants", func() { Expect(byName(view, "qwen3-8b-gguf-q8").MemoryBytes).To(Equal(gib(9))) }) - It("lets an authored min_memory win over the probe and skips that probe", func() { - base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq", MinMemory: "20GiB"}} - - view, err := gallery.DescribeVariants(models, base, probing(gib(24), map[string]uint64{ - "qwen3-8b-vllm-awq": gib(2), - })) - Expect(err).ToNot(HaveOccurred()) - Expect(byName(view, "qwen3-8b-vllm-awq").MemoryBytes).To(Equal(gib(20))) - Expect(probed).ToNot(ContainElement("qwen3-8b-vllm-awq")) - }) - It("reports a size it could not determine as unknown rather than as free", func() { // Zero on the wire means unknown. Rendering it as a zero-byte model // would advertise the largest download on offer as costless. diff --git a/core/gallery/models.go b/core/gallery/models.go index 9cc492125700..d1fbae9b62e9 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -105,9 +105,7 @@ func variantOptions(models []*GalleryModel, entry *GalleryModel, env ResolveEnv) } option := VariantOption{Variant: v, Backend: target.Backend} - // An authored figure short circuits the probe: the author already - // answered the question the probe would spend a round trip asking. - if v.MinMemory == "" && env.ProbeMemory != nil { + if env.ProbeMemory != nil { option.ProbedMemory = env.ProbeMemory(target) } options = append(options, option) @@ -206,7 +204,7 @@ func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, // checks, but a silent bypass makes a later out-of-memory failure // impossible to trace back to the pin, so it is recorded loudly here. if pin != "" { - if need, known, verr := selected.EffectiveMemory(); verr == nil && known && env.AvailableMemory < need { + if need, known := selected.EffectiveMemory(); known && env.AvailableMemory < need { xlog.Warn("Pinned model variant declares more memory than this system reports; installing anyway because the pin overrides hardware resolution", "model", entry.Name, "variant", selected.Variant.Model, "required_memory", need, "available_memory", env.AvailableMemory) } diff --git a/core/gallery/resolve_variant.go b/core/gallery/resolve_variant.go index cb111bc61c86..e63bbffb5033 100644 --- a/core/gallery/resolve_variant.go +++ b/core/gallery/resolve_variant.go @@ -31,9 +31,10 @@ type VariantOption struct { // must stay installable on every host and for every client. IsBase bool // ProbedMemory is the footprint measured live from the referenced entry's - // weights, in bytes. It is only consulted when the variant declares no - // min_memory of its own, because a human who measured a real load knows - // more than a pre-download estimate does. + // weights, in bytes. It is the only source of a variant's size. An author + // who needs to correct a bad estimate sets `size:` on the referenced entry, + // which the estimator behind the probe already prefers over its own + // guesswork, so the correction lands everywhere the size is used. // // Zero means the probe could not determine a size. That is an unknown, not // a zero requirement: a probe that cannot reach the network must never be @@ -42,17 +43,12 @@ type VariantOption struct { } // EffectiveMemory returns this option's memory requirement in bytes and whether -// one is known at all: the authored figure when there is one, else the live -// probe result, else nothing. -func (o VariantOption) EffectiveMemory() (uint64, bool, error) { - size, known, err := o.Variant.AuthoredMinMemory() - if err != nil || known { - return size, known, err - } +// one is known at all. +func (o VariantOption) EffectiveMemory() (uint64, bool) { if o.ProbedMemory > 0 { - return o.ProbedMemory, true, nil + return o.ProbedMemory, true } - return 0, false, nil + return 0, false } // ResolveEnv describes the host a variant is selected for. @@ -69,13 +65,12 @@ type ResolveEnv struct { // caller with no view of the hardware. BackendCompatible func(backend string) bool // ProbeMemory measures how much memory a referenced gallery entry needs, - // without downloading it. It is consulted only for variants that declare no - // min_memory, and a zero result means "could not tell", never "needs - // nothing". + // without downloading it. A zero result means "could not tell", never + // "needs nothing". // // It is a func field rather than a live network handle so specs can pin an // exact size, or an exact failure, without reaching the internet. A nil func - // leaves every unauthored variant unknown, which selection already handles. + // leaves every variant unknown, which selection already handles. // // SelectVariant never calls this: the install layer resolves every size into // VariantOption.ProbedMemory first, so the selector stays pure. @@ -149,10 +144,7 @@ func SelectVariant(options []VariantOption, env ResolveEnv, pin string) (Variant continue } - memory, known, err := o.EffectiveMemory() - if err != nil { - return VariantSelection{}, err - } + memory, known := o.EffectiveMemory() if known && memory > env.AvailableMemory { reasons = append(reasons, fmt.Sprintf("%s needs %s of memory", o.Variant.Model, humanBytes(memory))) continue diff --git a/core/gallery/resolve_variant_test.go b/core/gallery/resolve_variant_test.go index 49fcb3468ac5..3c3a0947496a 100644 --- a/core/gallery/resolve_variant_test.go +++ b/core/gallery/resolve_variant_test.go @@ -8,69 +8,49 @@ import ( ) var _ = Describe("VariantOption.EffectiveMemory", func() { - It("reports no requirement when nothing is authored and nothing was probed", func() { + It("reports no requirement when nothing was probed", func() { o := gallery.VariantOption{Variant: gallery.Variant{Model: "x"}} - size, known, err := o.EffectiveMemory() - Expect(err).ToNot(HaveOccurred()) + size, known := o.EffectiveMemory() Expect(known).To(BeFalse()) Expect(size).To(Equal(uint64(0))) }) - It("uses the probed size when nothing is authored", func() { + It("uses the probed size", func() { o := gallery.VariantOption{Variant: gallery.Variant{Model: "x"}, ProbedMemory: 6 * 1024 * 1024 * 1024} - size, known, err := o.EffectiveMemory() - Expect(err).ToNot(HaveOccurred()) + size, known := o.EffectiveMemory() Expect(known).To(BeTrue()) Expect(size).To(Equal(uint64(6 * 1024 * 1024 * 1024))) }) - It("lets an authored figure win over a probed one", func() { - // A human who measured a real load knows more than a pre-download - // estimate does, which is the entire reason min_memory survives. - o := gallery.VariantOption{ - Variant: gallery.Variant{Model: "x", MinMemory: "20GiB"}, - ProbedMemory: 6 * 1024 * 1024 * 1024, - } - size, known, err := o.EffectiveMemory() - Expect(err).ToNot(HaveOccurred()) - Expect(known).To(BeTrue()) - Expect(size).To(Equal(uint64(20 * 1024 * 1024 * 1024))) - }) - It("treats a failed probe as unknown rather than as a zero requirement", func() { // A probe that could not reach the network reports 0. Reading that as // "needs nothing" would make an unreachable host look like the perfect // fit and hand the user the largest download on offer. o := gallery.VariantOption{Variant: gallery.Variant{Model: "x"}, ProbedMemory: 0} - _, known, err := o.EffectiveMemory() - Expect(err).ToNot(HaveOccurred()) + _, known := o.EffectiveMemory() Expect(known).To(BeFalse()) }) - - It("errors on an unparseable authored figure rather than silently ignoring it", func() { - o := gallery.VariantOption{Variant: gallery.Variant{Model: "x", MinMemory: "twenty gigs"}} - _, _, err := o.EffectiveMemory() - Expect(err).To(HaveOccurred()) - }) }) var _ = Describe("SelectVariant", func() { gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 } - option := func(model, backend, minMemory string) gallery.VariantOption { + // Every size here is a probed one, because the probe is now the only source + // a variant's footprint can come from. A zero stands for the probe having + // been unable to tell, which is an unknown rather than a zero requirement. + option := func(model, backend string, probed uint64) gallery.VariantOption { return gallery.VariantOption{ - Variant: gallery.Variant{Model: model, MinMemory: minMemory}, - Backend: backend, + Variant: gallery.Variant{Model: model}, + Backend: backend, + ProbedMemory: probed, } } - // The base entry declares no memory requirement of its own, so a base - // option's size can only ever come from a probe. It is exempt from every - // filter regardless, which the fallback specs below pin down. + // The base is exempt from every filter, which the fallback specs below pin + // down; its size is carried only so ranking has nothing special to do. base := func(model string, probed uint64) gallery.VariantOption { - o := option(model, "llama-cpp", "") + o := option(model, "llama-cpp", probed) o.IsBase = true - o.ProbedMemory = probed return o } @@ -85,8 +65,8 @@ var _ = Describe("SelectVariant", func() { // The MLX build is both the largest and the only thing that would // otherwise win, so nothing but the backend gate can reject it. options := []gallery.VariantOption{ - option("m-mlx-8bit", "mlx", "24GiB"), - option("m-gguf-q8", "llama-cpp", "12GiB"), + option("m-mlx-8bit", "mlx", gib(24)), + option("m-gguf-q8", "llama-cpp", gib(12)), base("m-gguf-q4", gib(6)), } @@ -102,8 +82,8 @@ var _ = Describe("SelectVariant", func() { // The mirror image of the spec above, so the rejection is proven to // come from the host and not from something intrinsic to the entry. options := []gallery.VariantOption{ - option("m-mlx-8bit", "mlx", "24GiB"), - option("m-gguf-q8", "llama-cpp", "12GiB"), + option("m-mlx-8bit", "mlx", gib(24)), + option("m-gguf-q8", "llama-cpp", gib(12)), base("m-gguf-q4", gib(6)), } @@ -116,7 +96,7 @@ var _ = Describe("SelectVariant", func() { }) It("treats every backend as runnable when the host cannot be inspected", func() { - options := []gallery.VariantOption{option("m-mlx-8bit", "mlx", "24GiB"), base("m-gguf-q4", gib(6))} + options := []gallery.VariantOption{option("m-mlx-8bit", "mlx", gib(24)), base("m-gguf-q4", gib(6))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(80)}, "") Expect(err).ToNot(HaveOccurred()) @@ -129,9 +109,9 @@ var _ = Describe("SelectVariant", func() { // Authored smallest-first, so first-match would take m-q4 and any // ranking that ignores size would too. options := []gallery.VariantOption{ - option("m-q4", "llama-cpp", "6GiB"), - option("m-q8", "llama-cpp", "12GiB"), - option("m-f16", "llama-cpp", "24GiB"), + option("m-q4", "llama-cpp", gib(6)), + option("m-q8", "llama-cpp", gib(12)), + option("m-f16", "llama-cpp", gib(24)), base("m-base", gib(2)), } @@ -145,9 +125,9 @@ var _ = Describe("SelectVariant", func() { // Same set, authored largest-first. Order must make no difference at // all, which is the entire point of dropping ordered first-match. options := []gallery.VariantOption{ - option("m-f16", "llama-cpp", "24GiB"), - option("m-q8", "llama-cpp", "12GiB"), - option("m-q4", "llama-cpp", "6GiB"), + option("m-f16", "llama-cpp", gib(24)), + option("m-q8", "llama-cpp", gib(12)), + option("m-q4", "llama-cpp", gib(6)), base("m-base", gib(2)), } @@ -161,8 +141,8 @@ var _ = Describe("SelectVariant", func() { // nothing proves it does not fit, but it must never displace a // variant that is known to fit. options := []gallery.VariantOption{ - option("m-unknown", "llama-cpp", ""), - option("m-q8", "llama-cpp", "12GiB"), + option("m-unknown", "llama-cpp", 0), + option("m-q8", "llama-cpp", gib(12)), base("m-base", gib(2)), } @@ -173,7 +153,7 @@ var _ = Describe("SelectVariant", func() { It("keeps a variant of unknown size rather than dropping it", func() { options := []gallery.VariantOption{ - option("m-unknown", "llama-cpp", ""), + option("m-unknown", "llama-cpp", 0), base("m-base", gib(2)), } @@ -183,19 +163,13 @@ var _ = Describe("SelectVariant", func() { Expect(selection.FellBackToBase).To(BeFalse()) }) - It("ranks and filters a probed size exactly as an authored one", func() { - // Nothing here declares min_memory, so every figure came from the - // live probe. The probe is the primary source now and authoring the - // exception, so it has to drive both the filter and the ranking. - probed := func(model string, size uint64) gallery.VariantOption { - o := option(model, "llama-cpp", "") - o.ProbedMemory = size - return o - } + It("reports why a probed size that does not fit was rejected", func() { + // Ranking and filtering both run off the probe, so a rejection has to + // be traceable back to the figure the probe returned. options := []gallery.VariantOption{ - probed("m-q4", gib(6)), - probed("m-f16", gib(24)), - probed("m-q8", gib(12)), + option("m-q4", "llama-cpp", gib(6)), + option("m-f16", "llama-cpp", gib(24)), + option("m-q8", "llama-cpp", gib(12)), base("m-base", gib(2)), } @@ -206,7 +180,7 @@ var _ = Describe("SelectVariant", func() { }) It("admits a variant needing exactly the memory available", func() { - options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", gib(2))} + options := []gallery.VariantOption{option("m-q8", "llama-cpp", gib(12)), base("m-base", gib(2))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(12)}, "") Expect(err).ToNot(HaveOccurred()) @@ -217,8 +191,8 @@ var _ = Describe("SelectVariant", func() { Describe("falling back to the base", func() { It("selects the base when nothing else fits", func() { options := []gallery.VariantOption{ - option("m-q8", "llama-cpp", "12GiB"), - option("m-f16", "llama-cpp", "24GiB"), + option("m-q8", "llama-cpp", gib(12)), + option("m-f16", "llama-cpp", gib(24)), base("m-base", gib(2)), } @@ -234,7 +208,7 @@ var _ = Describe("SelectVariant", func() { // The base is exempt from the memory filter, not merely favoured by // it: there is nothing below it, so refusing here would make an entry // every older client installs fine uninstallable on newer ones. - options := []gallery.VariantOption{option("m-q8", "llama-cpp", "12GiB"), base("m-base", gib(2))} + options := []gallery.VariantOption{option("m-q8", "llama-cpp", gib(12)), base("m-base", gib(2))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: 0}, "") Expect(err).ToNot(HaveOccurred()) @@ -243,8 +217,8 @@ var _ = Describe("SelectVariant", func() { It("explains why each variant was rejected", func() { options := []gallery.VariantOption{ - option("m-mlx", "mlx", "8GiB"), - option("m-f16", "llama-cpp", "24GiB"), + option("m-mlx", "mlx", gib(8)), + option("m-f16", "llama-cpp", gib(24)), base("m-base", gib(2)), } @@ -258,7 +232,7 @@ var _ = Describe("SelectVariant", func() { }) It("errors when the caller supplies no base at all", func() { - options := []gallery.VariantOption{option("m-f16", "llama-cpp", "24GiB")} + options := []gallery.VariantOption{option("m-f16", "llama-cpp", gib(24))} _, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(4)}, "") Expect(err).To(MatchError(gallery.ErrNoVariantMatch)) @@ -268,7 +242,7 @@ var _ = Describe("SelectVariant", func() { Describe("explicit selection", func() { It("honors a pin the hardware would never have chosen", func() { options := []gallery.VariantOption{ - option("m-mlx", "mlx", "64GiB"), + option("m-mlx", "mlx", gib(64)), base("m-base", gib(2)), } @@ -281,7 +255,7 @@ var _ = Describe("SelectVariant", func() { }) It("honors a pin naming the base, declining an upgrade that fits", func() { - options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", gib(2))} + options := []gallery.VariantOption{option("m-f16", "llama-cpp", gib(8)), base("m-base", gib(2))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "m-base") Expect(err).ToNot(HaveOccurred()) @@ -289,7 +263,7 @@ var _ = Describe("SelectVariant", func() { }) It("matches a pin case-insensitively", func() { - options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", gib(2))} + options := []gallery.VariantOption{option("m-f16", "llama-cpp", gib(8)), base("m-base", gib(2))} selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "M-F16") Expect(err).ToNot(HaveOccurred()) @@ -297,19 +271,11 @@ var _ = Describe("SelectVariant", func() { }) It("fails loudly when the pin names nothing in the list", func() { - options := []gallery.VariantOption{option("m-f16", "llama-cpp", "8GiB"), base("m-base", gib(2))} + options := []gallery.VariantOption{option("m-f16", "llama-cpp", gib(8)), base("m-base", gib(2))} _, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(64)}, "m-gone") Expect(err).To(MatchError(gallery.ErrPinNotFound)) Expect(err.Error()).To(ContainSubstring("m-gone")) }) }) - - It("propagates an unparseable figure instead of treating it as unconstrained", func() { - options := []gallery.VariantOption{option("bad", "llama-cpp", "lots"), base("m-base", gib(2))} - - _, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(8)}, "") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("bad")) - }) }) diff --git a/core/gallery/variant.go b/core/gallery/variant.go index 35ca8bfb7b93..ad7ff093263b 100644 --- a/core/gallery/variant.go +++ b/core/gallery/variant.go @@ -1,44 +1,18 @@ package gallery -import ( - "fmt" - - "github.com/mudler/LocalAI/pkg/vram" -) - // Variant is one option in a gallery entry's variant list. It references an // existing gallery entry by name, and that is all an author has to write. // // Authored order carries no meaning. Selection filters out what this host // cannot run and then ranks what is left, so hardware knowledge lives in the // selector rather than being pushed onto whoever edits the gallery. +// +// There is deliberately no authored memory figure here. When the live probe +// misreads a variant's footprint, the fix is the referenced entry's own `size:` +// field: the estimator already prefers a declared size over its own guesswork, +// so correcting it there fixes the figure for every consumer rather than only +// for this one selection pass. type Variant struct { // Model is the name of a gallery entry that declares no variants of its own. Model string `json:"model" yaml:"model"` - // MinMemory is the authored memory requirement (e.g. "20GiB"). It exists to - // override the size the installer probes live from the model's weights when - // that figure is known to be wrong; most variants should not need it. - // - // One number covers both VRAM and system RAM. A model's footprint is - // roughly the same wherever it lives, and the selector compares this against - // whichever of the two this host will actually use, so splitting it in two - // would only invite the pair to disagree. - MinMemory string `json:"min_memory,omitempty" yaml:"min_memory,omitempty"` -} - -// AuthoredMinMemory returns the hand-written memory requirement in bytes and -// whether one was written at all. -// -// An absent requirement is not a zero requirement. Callers must not read the -// returned 0 as "fits anywhere"; it means the author said nothing, and the -// figure has to come from a live probe instead. -func (v Variant) AuthoredMinMemory() (uint64, bool, error) { - if v.MinMemory == "" { - return 0, false, nil - } - size, err := vram.ParseSizeString(v.MinMemory) - if err != nil { - return 0, false, fmt.Errorf("variant %q has an unparseable min_memory %q: %w", v.Model, v.MinMemory, err) - } - return size, true, nil } diff --git a/core/gallery/variants_install_test.go b/core/gallery/variants_install_test.go index 8bf275f33ae8..e67faaecfa09 100644 --- a/core/gallery/variants_install_test.go +++ b/core/gallery/variants_install_test.go @@ -36,19 +36,16 @@ url: "github:example/repo/qwen3.6-27b.yaml@master" variants: - model: qwen3.6-27b-mlx-8bit - model: qwen3.6-27b-gguf-q8 - min_memory: 28GiB `), &m) Expect(err).ToNot(HaveOccurred()) Expect(m.Name).To(Equal("qwen3.6-27b")) Expect(m.URL).To(Equal("github:example/repo/qwen3.6-27b.yaml@master")) Expect(m.HasVariants()).To(BeTrue()) Expect(m.Variants).To(HaveLen(2)) - // The first variant is nothing but a name, which is the shape authoring - // is meant to reach for. + // A variant is nothing but a name, which is the whole of the authoring + // surface. Expect(m.Variants[0].Model).To(Equal("qwen3.6-27b-mlx-8bit")) - Expect(m.Variants[0].MinMemory).To(BeEmpty()) Expect(m.Variants[1].Model).To(Equal("qwen3.6-27b-gguf-q8")) - Expect(m.Variants[1].MinMemory).To(Equal("28GiB")) }) }) @@ -79,12 +76,24 @@ var _ = Describe("ResolveVariant", func() { base = newModel("qwen3-8b-gguf-q4", "file://gguf.yaml", "Qwen3 8B Q4", "qwen.png") base.Backend = "llama-cpp" base.Tags = []string{"llm"} - base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq", MinMemory: "20GiB"}} + base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq"}} models = []*gallery.GalleryModel{upgrade, base} }) + // The AWQ build measures 20GiB, so the specs below vary only the host and + // read off which payload resolution lands on. The probe is injected rather + // than performed, so nothing here reaches the network. env := func(memory uint64) gallery.ResolveEnv { - return gallery.ResolveEnv{AvailableMemory: memory, BackendCompatible: runsEverything} + return gallery.ResolveEnv{ + AvailableMemory: memory, + BackendCompatible: runsEverything, + ProbeMemory: func(target *gallery.GalleryModel) uint64 { + if target.Name == "qwen3-8b-vllm-awq" { + return 20 * 1024 * 1024 * 1024 + } + return 0 + }, + } } It("installs a fitting variant's payload under the entry's name", func() { @@ -141,12 +150,6 @@ var _ = Describe("ResolveVariant", func() { return e } - BeforeEach(func() { - // No authored figure anywhere, so every size below comes from the - // probe. This is the shape authoring is meant to reach for. - base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq"}} - }) - It("selects a variant the probe shows to fit", func() { resolved, variant, err := gallery.ResolveVariant(models, base, probing(gib(24), map[string]uint64{"qwen3-8b-vllm-awq": gib(20)}), "") @@ -188,23 +191,6 @@ var _ = Describe("ResolveVariant", func() { Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4")) Expect(resolved.URL).To(Equal("file://gguf.yaml")) }) - - It("does not probe a variant that declares its own min_memory", func() { - // Probing costs a network round trip, so an authored figure has to - // suppress it outright rather than merely outrank it. - base.Variants = []gallery.Variant{{Model: "qwen3-8b-vllm-awq", MinMemory: "20GiB"}} - probed := []string{} - e := env(gib(24)) - e.ProbeMemory = func(target *gallery.GalleryModel) uint64 { - probed = append(probed, target.Name) - return gib(80) - } - - _, variant, err := gallery.ResolveVariant(models, base, e, "") - Expect(err).ToNot(HaveOccurred()) - Expect(probed).To(BeEmpty()) - Expect(variant.Model).To(Equal("qwen3-8b-vllm-awq")) - }) }) It("presents the entry's metadata, not the variant's", func() { @@ -350,6 +336,16 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { return m } + // sizedEntry declares its footprint through the entry's own size:, which is + // the only way an author influences a variant's measured size. The estimator + // prefers a declared size over its own guesswork and parses it locally, so + // these specs pin an exact figure without touching the network. + sizedEntry := func(name, backend, size string) gallery.GalleryModel { + m := entry(name, backend) + m.Size = size + return m + } + // urlEntry describes an entry through a url rather than an inline // config_file. The distinction matters for the recorded name: the url branch // reads a name out of the fetched config, and that name is the referenced @@ -375,9 +371,10 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { } // withVariants attaches alternative builds to an otherwise ordinary entry. - // The figures are absolute rather than relative to the host: "0GiB" always - // fits and "10000GiB" never does, so these specs assert on selection rather - // than on whatever memory the machine running them happens to have. + // Where a spec needs a definite size it declares it on the referenced entry + // with sizedEntry, in absolute terms rather than relative to the host, so + // these specs assert on selection rather than on whatever memory the machine + // running them happens to have. withVariants := func(m gallery.GalleryModel, variants ...gallery.Variant) gallery.GalleryModel { m.Variants = variants return m @@ -410,8 +407,8 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { It("installs the entry's own payload when no variant fits the host", func() { newGallery( withVariants(entry("qwen3-8b-q4", "base-backend"), - gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "10000GiB"}), - entry("qwen3-8b-q8", "upgrade-backend"), + gallery.Variant{Model: "qwen3-8b-q8"}), + sizedEntry("qwen3-8b-q8", "upgrade-backend", "10000GiB"), ) // No machine clears 10000GiB, so this asserts the base is the last @@ -423,8 +420,8 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { It("installs a fitting variant's payload under the entry's own name", func() { newGallery( withVariants(entry("qwen3-8b-q4", "base-backend"), - gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}), - entry("qwen3-8b-q8", "upgrade-backend"), + gallery.Variant{Model: "qwen3-8b-q8"}), + sizedEntry("qwen3-8b-q8", "upgrade-backend", "16MiB"), ) Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) @@ -454,11 +451,11 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { // would take the small one. newGallery( withVariants(entry("qwen3-8b-q4", "base-backend"), - gallery.Variant{Model: "qwen3-8b-small", MinMemory: "0GiB"}, - gallery.Variant{Model: "qwen3-8b-large", MinMemory: "1KiB"}, + gallery.Variant{Model: "qwen3-8b-small"}, + gallery.Variant{Model: "qwen3-8b-large"}, ), - entry("qwen3-8b-small", "small-backend"), - entry("qwen3-8b-large", "large-backend"), + sizedEntry("qwen3-8b-small", "small-backend", "16MiB"), + sizedEntry("qwen3-8b-large", "large-backend", "256MiB"), ) Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) @@ -474,7 +471,7 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { // the only thing that can send this to the base. newGallery( withVariants(entry("qwen3-8b-q4", "base-backend"), - gallery.Variant{Model: "qwen3-8b-mlx", MinMemory: "0GiB"}), + gallery.Variant{Model: "qwen3-8b-mlx"}), entry("qwen3-8b-mlx", "mlx"), ) @@ -489,7 +486,7 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { // anything reads the record back. newGallery( withVariants(urlEntry("qwen3-8b-q4", "base-backend"), - gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}), + gallery.Variant{Model: "qwen3-8b-q8"}), urlEntry("qwen3-8b-q8", "upgrade-backend"), ) @@ -508,7 +505,7 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { It("refuses a variant name the entry does not declare, naming what was asked for", func() { newGallery( withVariants(entry("qwen3-8b-q4", "base-backend"), - gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}), + gallery.Variant{Model: "qwen3-8b-q8"}), entry("qwen3-8b-q8", "upgrade-backend"), ) @@ -538,7 +535,7 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { It("records a pin and honors it on a plain reinstall", func() { newGallery( withVariants(entry("qwen3-8b-q4", "base-backend"), - gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}), + gallery.Variant{Model: "qwen3-8b-q8"}), entry("qwen3-8b-q8", "upgrade-backend"), ) @@ -563,7 +560,7 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { It("honors a pin recorded under a custom install name", func() { newGallery( withVariants(entry("qwen3-8b-q4", "base-backend"), - gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}), + gallery.Variant{Model: "qwen3-8b-q8"}), entry("qwen3-8b-q8", "upgrade-backend"), ) @@ -586,7 +583,7 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { It("writes each declared url only once into the persisted gallery file", func() { base := withVariants(entry("qwen3-8b-q4", "base-backend"), - gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}) + gallery.Variant{Model: "qwen3-8b-q8"}) base.URLs = []string{"https://example.invalid/qwen3"} newGallery(base, entry("qwen3-8b-q8", "upgrade-backend")) diff --git a/core/gallery/variants_lint_test.go b/core/gallery/variants_lint_test.go index a60de7783861..8e7aef94dbcd 100644 --- a/core/gallery/variants_lint_test.go +++ b/core/gallery/variants_lint_test.go @@ -75,28 +75,6 @@ func checkVariantReferences(entries []gallery.GalleryModel) []variantViolation { return violations } -// checkVariantMemory verifies every authored memory figure parses. -// -// Absence is fine and deliberately not flagged: a variant that declares nothing -// gets its size from a live probe, and even a probe that fails leaves a -// legitimate unknown that selection handles by ranking the variant last. An -// UNPARSEABLE figure is different, because it makes selection fail outright for -// the whole entry rather than degrade. -func checkVariantMemory(entries []gallery.GalleryModel) []variantViolation { - var violations []variantViolation - for _, e := range entries { - if !e.HasVariants() { - continue - } - for _, v := range e.Variants { - if _, _, err := v.AuthoredMinMemory(); err != nil { - violations = append(violations, variantViolation{Entry: e.Name, Variant: v.Model, Detail: "has a bad min_memory: " + err.Error()}) - } - } - } - return violations -} - // loadGalleryIndex parses gallery/index.yaml once for the whole suite. The // index carries well over a thousand entries, so re-parsing it per spec is // pure overhead. @@ -145,11 +123,13 @@ var _ = Describe("gallery variant lint helpers", func() { }) It("passes every invariant on a valid entry", func() { + // Authoring is nothing but a list of names, so a bare list of them must + // be entirely valid. entries := variantFixture( entryWithVariants("base", "u://base", - gallery.Variant{Model: "big", MinMemory: "24GiB"}, - gallery.Variant{Model: "mid", MinMemory: "12GiB"}, - gallery.Variant{Model: "metal-big", MinMemory: "32GiB"}, + gallery.Variant{Model: "big"}, + gallery.Variant{Model: "mid"}, + gallery.Variant{Model: "metal-big"}, gallery.Variant{Model: "small"}, ), plainEntry("big", "u://big"), @@ -159,19 +139,6 @@ var _ = Describe("gallery variant lint helpers", func() { ) Expect(checkVariantReferences(entries)).To(BeEmpty()) - Expect(checkVariantMemory(entries)).To(BeEmpty()) - }) - - It("passes every invariant on an entry that declares no memory at all", func() { - // Authoring is meant to be nothing but a list of names, so an entry - // whose variants carry no figures must be entirely valid. - entries := variantFixture( - entryWithVariants("base", "u://base", gallery.Variant{Model: "big"}), - plainEntry("big", "u://big"), - ) - - Expect(checkVariantReferences(entries)).To(BeEmpty()) - Expect(checkVariantMemory(entries)).To(BeEmpty()) }) Describe("checkVariantReferences", func() { @@ -212,31 +179,6 @@ var _ = Describe("gallery variant lint helpers", func() { }) }) - Describe("checkVariantMemory", func() { - It("flags a variant whose memory figure cannot be parsed", func() { - entries := variantFixture( - entryWithVariants("base", "u://base", gallery.Variant{Model: "a", MinMemory: "lots"}), - plainEntry("a", "u://a"), - ) - - violations := checkVariantMemory(entries) - Expect(violations).To(HaveLen(1)) - Expect(violations[0].Variant).To(Equal("a")) - Expect(violations[0].Detail).To(ContainSubstring("bad min_memory")) - }) - - It("accepts a variant that declares no figure at all", func() { - // The overwhelmingly common shape: a bare name, whose size the - // installer probes live. Flagging it would make the rule reject the - // authoring style the design exists to enable. - entries := variantFixture( - entryWithVariants("base", "u://base", gallery.Variant{Model: "a"}), - plainEntry("a", "u://a"), - ) - - Expect(checkVariantMemory(entries)).To(BeEmpty()) - }) - }) }) var _ = Describe("gallery/index.yaml variant invariants", Ordered, func() { @@ -255,9 +197,4 @@ var _ = Describe("gallery/index.yaml variant invariants", Ordered, func() { v := checkVariantReferences(entries) Expect(v).To(BeEmpty(), formatViolations(v)) }) - - It("declares only parseable memory figures", func() { - v := checkVariantMemory(entries) - Expect(v).To(BeEmpty(), formatViolations(v)) - }) }) diff --git a/core/schema/gallery-model.schema.json b/core/schema/gallery-model.schema.json index 0cf0184d1335..0471f6dce8e9 100644 --- a/core/schema/gallery-model.schema.json +++ b/core/schema/gallery-model.schema.json @@ -67,10 +67,6 @@ "model": { "type": "string", "description": "Name of a gallery entry that declares no variants of its own" - }, - "min_memory": { - "type": "string", - "description": "Authored memory requirement, for example 20GiB. Optional, and only needed to override the size the installer probes live from the model's weights when that figure is wrong. Authoritative: an authored figure suppresses the probe entirely." } }, "additionalProperties": false diff --git a/core/services/galleryop/model_variant_test.go b/core/services/galleryop/model_variant_test.go index ada429130b3c..689a2ecc442d 100644 --- a/core/services/galleryop/model_variant_test.go +++ b/core/services/galleryop/model_variant_test.go @@ -62,10 +62,12 @@ var _ = Describe("LocalModelManager variant selection", func() { Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { Expect(os.RemoveAll(modelsDir)).To(Succeed()) }) - // "0GiB" always fits, so auto-selection would take the upgrade on any - // machine. Only an honored pin can land the install on the base. + // The upgrade declares no size and carries no weight files, so the probe + // reports an unknown, the variant survives the filter and auto-selection + // would take it on any machine. Only an honored pin can land the install + // on the base. entries := []gallery.GalleryModel{ - entry("qwen3-8b-q4", "base-backend", gallery.Variant{Model: "qwen3-8b-q8", MinMemory: "0GiB"}), + entry("qwen3-8b-q4", "base-backend", gallery.Variant{Model: "qwen3-8b-q8"}), entry("qwen3-8b-q8", "upgrade-backend"), } out, err := yaml.Marshal(entries) From 9a43a27c2ac72567bcd32c2bcd4375723cc33587 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 18 Jul 2026 23:49:14 +0000 Subject: [PATCH 20/48] fix(gallery): rank the entry's own build against its variants Variant selection pulled the declaring entry's own payload, the base, out of the candidate set and consulted it only once every declared variant had been rejected. Two real failures followed. A variant whose size the probe cannot determine deliberately survives the memory filter, because nothing proves it does not fit. As the only survivor it then won outright on any host, however small: a 2GiB machine installed an unmeasured variant in preference to the 4GiB build the entry itself ships, with no warning. 241 of the 1280 current index entries carry no files and no size, which is exactly that shape. "Largest wins" also broke whenever the base was the largest. An author writing a Q8 entry that offers a Q4 downgrade for small hosts, a natural shape that nothing in the lint, schema or docs discourages, had the Q4 installed on every large host instead. Make the base an ordinary participant. It is still exempt from both filters, so selection always terminates on something installable, but it is now ranked against the variants: a proven fit first and largest, then the base, then any variant whose size nothing could measure. Both failures disappear together. The base is probed for its size accordingly, which it was not before, because an unsized base would lose every contest to an unmeasurable variant. FellBackToBase is kept but narrowed to "no declared variant survived", rather than "the base was chosen", since the base now also wins on merit and that is not worth warning about. A recalled variant pin also became a permanent install failure. A pin the caller supplies on this request must stay fatal, but one recalled from ._gallery_.yaml can be invalidated by any later gallery edit, and failing on it turned one rename into a model that could never be reinstalled or upgraded again short of deleting a dotfile the user has never heard of. A stale recalled pin is now dropped with a warning naming it, and selection runs as if it had never been recorded. Also drop the last textual reference to two abandoned designs from the DetectedCapability comment, correct the documented variants JSON example, which showed a memory_bytes of 0 that omitempty makes impossible, and remove an em dash from the install skill. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- .agents/adding-gallery-models.md | 10 +- core/gallery/describe_variants.go | 7 +- core/gallery/models.go | 43 ++++-- core/gallery/resolve_variant.go | 129 ++++++++++++------ core/gallery/resolve_variant_test.go | 74 +++++++++- core/gallery/variants_install_test.go | 101 ++++++++++++-- core/services/galleryop/model_variant_test.go | 13 +- docs/content/features/model-gallery.md | 24 +++- .../prompts/skills/install_chat_model.md | 2 +- pkg/system/capabilities.go | 6 +- 10 files changed, 322 insertions(+), 87 deletions(-) diff --git a/.agents/adding-gallery-models.md b/.agents/adding-gallery-models.md index 93563c76279d..aef70e060e8d 100644 --- a/.agents/adding-gallery-models.md +++ b/.agents/adding-gallery-models.md @@ -114,11 +114,15 @@ Rules: and must not declare `variants` of its own. - **Order carries no meaning.** Do not try to encode a preference; write the list in whatever order reads best. +- **A variant may be smaller than the declaring entry.** Offering a downgrade + for small hosts is a normal shape: the declaring entry's own build competes on + size like every other candidate, so a large host keeps the large build. - **Do not describe hardware.** At install time LocalAI drops variants whose backend cannot run on the host, drops those that do not fit available memory, - and installs the largest survivor, falling back to the declaring entry's own - build. Sizes are measured live from the weights and cached, so nothing has to - be written down. + and installs the largest of what is left, the declaring entry's own build + included. That build is never dropped, so selection always terminates on + something installable. Sizes are measured live from the weights and cached, so + nothing has to be written down. - A variant is nothing but a name; there is no per-variant memory field. When the measured size for a build is wrong, correct it on the referenced entry by setting that entry's own `size:` (e.g. `size: "20GiB"`). The estimator prefers diff --git a/core/gallery/describe_variants.go b/core/gallery/describe_variants.go index 726308909bab..b7bf52b88812 100644 --- a/core/gallery/describe_variants.go +++ b/core/gallery/describe_variants.go @@ -12,9 +12,10 @@ type VariantView struct { // it, and it is also the reason Fits may be false on a host whose memory // would be ample. Backend string `json:"backend"` - // MemoryBytes is the measured footprint, or 0 when it could not be - // determined. Zero is unknown, never "needs nothing", so a client must - // render it as unknown rather than as a free option. + // MemoryBytes is the measured footprint. It is omitted from the JSON + // entirely when the size could not be determined, rather than serialized as + // a zero that a client would have to know to read as unknown. An absent key + // never means "needs nothing". MemoryBytes uint64 `json:"memory_bytes,omitempty"` // Fits reports whether auto-selection would consider this variant on this // host: its backend can run here and its known footprint is within budget. diff --git a/core/gallery/models.go b/core/gallery/models.go index d1fbae9b62e9..18a314dfca40 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -111,13 +111,18 @@ func variantOptions(models []*GalleryModel, entry *GalleryModel, env ResolveEnv) options = append(options, option) } - // The base is never filtered nor ranked, so its size would change nothing - // and probing it would spend a round trip on a discarded answer. - return append(options, VariantOption{ + // The base is probed like any other candidate. It is never filtered out, but + // it IS ranked against the variants, and an unsized base would lose every + // contest to a variant whose size nothing could measure. + base := VariantOption{ Variant: Variant{Model: entry.Name}, Backend: entry.Backend, IsBase: true, - }), nil + } + if env.ProbeMemory != nil { + base.ProbedMemory = env.ProbeMemory(entry) + } + return append(options, base), nil } // probeContextLength is the context size the footprint estimate is taken at. @@ -180,9 +185,10 @@ func probeEntryMemory(ctx context.Context, entry *GalleryModel) uint64 { // // The base entry always resolves, whatever the host has. It is a complete entry // that every older LocalAI release installs unconditionally, so refusing it here -// would make the gallery behave worse the newer the client is. That is also why -// the base declares no memory requirement of its own: a floor that can only -// warn cannot change any outcome. +// would make the gallery behave worse the newer the client is. Being exempt from +// the filters does not make it exempt from ranking: it is measured and compared +// like every declared variant, and wins by default only once they have all been +// ruled out. func ResolveVariant(models []*GalleryModel, entry *GalleryModel, env ResolveEnv, pin string) (*GalleryModel, Variant, error) { options, err := variantOptions(models, entry, env) if err != nil { @@ -462,19 +468,40 @@ func InstallModelFromGallery( // back under the entry's own name would miss the record for every custom-named // install and silently re-resolve a deliberately pinned model onto a // different variant, possibly swapping its backend. + // + // recalledPin tracks where the pin came from, because the two sources must + // fail differently when the name no longer resolves. See below. + recalledPin := "" if pin == "" { installName := model.Name if req.Name != "" { installName = req.Name } if previous, err := GetLocalModelConfiguration(systemState.Model.ModelsPath, installName); err == nil && previous != nil { - pin = previous.PinnedVariant + recalledPin = previous.PinnedVariant + pin = recalledPin } } env := HostResolveEnv(ctx, systemState) resolved, variant, err := ResolveVariant(models, model, env, pin) + // A pin the caller supplied on this request must stay fatal: they named + // something this entry cannot give, and installing anything else would + // report success for a request that was not honored. + // + // A pin recalled from disk is different. The gallery can rename or withdraw + // a variant long after it was pinned, and the user is not asking for it + // again on this call. Failing here would turn one gallery edit into a + // permanently unrepairable model whose only remedy is deleting a dotfile + // they have never heard of, so the stale pin is dropped and selection runs + // as if it had never been recorded. + if err != nil && recalledPin != "" && errors.Is(err, ErrPinNotFound) { + xlog.Warn("The recorded variant pin for this model no longer exists in the gallery; re-selecting automatically", + "model", model.Name, "dropped_pin", recalledPin) + pin = "" + resolved, variant, err = ResolveVariant(models, model, env, "") + } if err != nil { return err } diff --git a/core/gallery/resolve_variant.go b/core/gallery/resolve_variant.go index e63bbffb5033..34363b55a8e4 100644 --- a/core/gallery/resolve_variant.go +++ b/core/gallery/resolve_variant.go @@ -27,8 +27,10 @@ type VariantOption struct { // precisely why a gallery author never has to describe hardware. Backend string // IsBase marks the declaring entry's own payload. It is exempt from every - // filter and is the answer when nothing else survives, because the entry - // must stay installable on every host and for every client. + // filter, because the entry must stay installable on every host and for + // every client, but it is otherwise an ordinary candidate: it is ranked + // against the declared variants rather than consulted only once they have + // all been rejected. IsBase bool // ProbedMemory is the footprint measured live from the referenced entry's // weights, in bytes. It is the only source of a variant's size. An author @@ -87,9 +89,14 @@ func (e ResolveEnv) backendRuns(backend string) bool { // VariantSelection is the outcome of a selection pass. type VariantSelection struct { Option VariantOption - // FellBackToBase reports that no declared variant survived and the entry's - // own payload was chosen instead. Callers log this, because a host quietly - // taking the base when upgrades were on offer is worth being able to see. + // FellBackToBase reports that no declared variant survived the filters and + // the entry's own payload was all that remained. Callers log this, because a + // host quietly taking the base when upgrades were on offer is worth being + // able to see. + // + // It is deliberately narrower than "the base was selected": the base also + // wins on merit whenever it outranks every surviving variant, and that is an + // ordinary, uninteresting outcome rather than something to warn about. FellBackToBase bool // Reasons explains, one line per rejected variant, why it was dropped. Reasons []string @@ -107,11 +114,13 @@ type VariantSelection struct { // dropped. A variant with an UNKNOWN requirement survives, because nothing // proves it does not fit and refusing on a size the probe could not read // would let a network hiccup silently downgrade what gets installed. -// 4. The largest survivor wins. A bigger footprint is a higher quality -// quantization of the same model, so among things that fit, more is better. -// Unknown requirements rank last, so a proven fit always beats a guess. -// 5. With no survivor the base option wins. The base always installs; this -// never refuses. +// 4. The base survives both filters unconditionally, and then competes. It is +// a candidate like any other, not a last resort: an entry whose own build is +// the largest thing that fits must win against a smaller variant, and a base +// of known size must win against a variant whose size nothing could measure. +// 5. The survivors are ranked and the best one wins, by rankOf below. +// 6. With no survivor at all, which can only happen when the caller supplied no +// base, there is nothing to install and this reports ErrNoVariantMatch. func SelectVariant(options []VariantOption, env ResolveEnv, pin string) (VariantSelection, error) { if pin != "" { for _, o := range options { @@ -125,54 +134,88 @@ func SelectVariant(options []VariantOption, env ResolveEnv, pin string) (Variant type ranked struct { option VariantOption memory uint64 - known bool + rank int } - var base *VariantOption survivors := make([]ranked, 0, len(options)) reasons := make([]string, 0, len(options)) + survivingVariants := 0 for i := range options { o := options[i] - if o.IsBase { - base = &options[i] - continue - } - - if !env.backendRuns(o.Backend) { - reasons = append(reasons, fmt.Sprintf("%s needs backend %q, which cannot run on this system", o.Variant.Model, o.Backend)) - continue - } - memory, known := o.EffectiveMemory() - if known && memory > env.AvailableMemory { - reasons = append(reasons, fmt.Sprintf("%s needs %s of memory", o.Variant.Model, humanBytes(memory))) - continue + + // The base skips both gates. There is nothing below it, so refusing it + // would make an entry every older LocalAI installs fine uninstallable on + // newer ones. + if !o.IsBase { + if !env.backendRuns(o.Backend) { + reasons = append(reasons, fmt.Sprintf("%s needs backend %q, which cannot run on this system", o.Variant.Model, o.Backend)) + continue + } + if known && memory > env.AvailableMemory { + reasons = append(reasons, fmt.Sprintf("%s needs %s of memory", o.Variant.Model, humanBytes(memory))) + continue + } + survivingVariants++ } - survivors = append(survivors, ranked{option: o, memory: memory, known: known}) + survivors = append(survivors, ranked{option: o, memory: memory, rank: rankOf(o, env)}) } - if len(survivors) > 0 { - // Stable so that variants with identical requirements keep their - // authored order, which is the only thing order still decides. - sort.SliceStable(survivors, func(i, j int) bool { - if survivors[i].known != survivors[j].known { - return survivors[i].known - } - return survivors[i].memory > survivors[j].memory - }) - return VariantSelection{Option: survivors[0].option, Reasons: reasons}, nil + if len(survivors) == 0 { + return VariantSelection{}, fmt.Errorf( + "%w: %s of memory available; variants: %s", + ErrNoVariantMatch, humanBytes(env.AvailableMemory), strings.Join(reasons, "; "), + ) } - if base != nil { - return VariantSelection{Option: *base, FellBackToBase: true, Reasons: reasons}, nil - } + // Stable so that options within one rank and of identical size keep their + // authored order, which is the only thing order still decides. + sort.SliceStable(survivors, func(i, j int) bool { + if survivors[i].rank != survivors[j].rank { + return survivors[i].rank < survivors[j].rank + } + return survivors[i].memory > survivors[j].memory + }) + + winner := survivors[0].option + return VariantSelection{ + Option: winner, + // Only a base that won by default is worth reporting. A base that + // outranked live competition is an ordinary selection. + FellBackToBase: winner.IsBase && survivingVariants == 0, + Reasons: reasons, + }, nil +} - return VariantSelection{}, fmt.Errorf( - "%w: %s of memory available; variants: %s", - ErrNoVariantMatch, humanBytes(env.AvailableMemory), strings.Join(reasons, "; "), - ) +// Ranks, best first. Within a rank the larger footprint wins, because a bigger +// build is a higher quality quantization of the same model. +const ( + // rankProvenFit is a measured size that the host is measured to satisfy. + rankProvenFit = iota + // rankBase is the entry's own build when it is not a proven fit: either it + // needs more memory than the host reports, or its size could not be + // measured either. It still outranks any unsized variant, because it is the + // payload the entry is guaranteed to be able to install and a variant of + // unmeasurable size is a guess. Taking the guess on a host that cannot be + // shown to accommodate it is how an unreachable network silently changes + // what gets installed. + rankBase + // rankUnknownFit is a variant whose size nothing could measure. Nothing + // proves it does not fit, so it is not dropped, but nothing proves it does + // either, so it ranks last. + rankUnknownFit +) + +func rankOf(o VariantOption, env ResolveEnv) int { + if memory, known := o.EffectiveMemory(); known && memory <= env.AvailableMemory { + return rankProvenFit + } + if o.IsBase { + return rankBase + } + return rankUnknownFit } func humanBytes(b uint64) string { diff --git a/core/gallery/resolve_variant_test.go b/core/gallery/resolve_variant_test.go index 3c3a0947496a..08713d9c0df4 100644 --- a/core/gallery/resolve_variant_test.go +++ b/core/gallery/resolve_variant_test.go @@ -47,7 +47,8 @@ var _ = Describe("SelectVariant", func() { } // The base is exempt from every filter, which the fallback specs below pin - // down; its size is carried only so ranking has nothing special to do. + // down, but it is ranked against the variants like any other candidate, so + // its size is load-bearing. base := func(model string, probed uint64) gallery.VariantOption { o := option(model, "llama-cpp", probed) o.IsBase = true @@ -159,8 +160,62 @@ var _ = Describe("SelectVariant", func() { selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(4)}, "") Expect(err).ToNot(HaveOccurred()) - Expect(selection.Option.Variant.Model).To(Equal("m-unknown")) + // Surviving is observable through the rejection reasons: a dropped + // variant is always accounted for there, and this one is not. + Expect(selection.Reasons).ToNot(ContainElement(ContainSubstring("m-unknown"))) + // It survives, but it does not win: the base is a sized, guaranteed + // payload and an unmeasurable variant is a guess. + Expect(selection.Option.Variant.Model).To(Equal("m-base")) + Expect(selection.FellBackToBase).To(BeFalse()) + }) + + It("installs the base rather than an unsized variant on a host too small for either", func() { + // The exact shape 241 of the current index entries have: a referenced + // entry with no files and no size, whose probe can only answer + // "unknown". Ranking it above the base would install an unmeasured + // download on a machine with 2GiB, and would do so silently. + options := []gallery.VariantOption{ + option("m-unknown", "llama-cpp", 0), + base("m-base-q4", gib(4)), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(2)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-base-q4")) + Expect(selection.Option.IsBase).To(BeTrue()) + }) + + It("selects the base when the base is the largest option that fits", func() { + // A Q8 base offering a Q4 downgrade for small hosts is a natural + // authoring shape. Treating the base as a last resort would install + // the Q4 on every host large enough for the Q8 and permanently + // downgrade the user. + options := []gallery.VariantOption{ + option("m-q4", "llama-cpp", gib(4)), + base("m-base-q8", gib(8)), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(16)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-base-q8")) + // The Q4 survived every filter, so this is the base winning on rank + // and not the base being fallen back to. Expect(selection.FellBackToBase).To(BeFalse()) + Expect(selection.Reasons).To(BeEmpty()) + }) + + It("selects a smaller variant when the base does not fit but the variant does", func() { + // The mirror of the spec above: the base competes, it does not win by + // default, so a host that cannot hold it must still take the downgrade + // the entry offers for exactly that case. + options := []gallery.VariantOption{ + option("m-q4", "llama-cpp", gib(4)), + base("m-base-q8", gib(8)), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(6)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-q4")) }) It("reports why a probed size that does not fit was rejected", func() { @@ -213,6 +268,21 @@ var _ = Describe("SelectVariant", func() { selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: 0}, "") Expect(err).ToNot(HaveOccurred()) Expect(selection.Option.Variant.Model).To(Equal("m-base")) + Expect(selection.Option.IsBase).To(BeTrue()) + Expect(selection.FellBackToBase).To(BeTrue()) + }) + + It("prefers the base to an unsized variant even when the base itself is unsized", func() { + // Neither can be shown to fit, so nothing separates them on size. The + // base is still the payload the entry is guaranteed to install. + options := []gallery.VariantOption{ + option("m-unknown", "llama-cpp", 0), + base("m-base", 0), + } + + selection, err := gallery.SelectVariant(options, gallery.ResolveEnv{AvailableMemory: gib(8)}, "") + Expect(err).ToNot(HaveOccurred()) + Expect(selection.Option.Variant.Model).To(Equal("m-base")) }) It("explains why each variant was rejected", func() { diff --git a/core/gallery/variants_install_test.go b/core/gallery/variants_install_test.go index e67faaecfa09..900839c5251e 100644 --- a/core/gallery/variants_install_test.go +++ b/core/gallery/variants_install_test.go @@ -2,6 +2,7 @@ package gallery_test import ( "context" + "fmt" "os" "path/filepath" "runtime" @@ -168,15 +169,17 @@ var _ = Describe("ResolveVariant", func() { Expect(resolved.URL).To(Equal("file://gguf.yaml")) }) - It("completes the install when the probe cannot determine a size", func() { - // A failed probe reports 0. That is an unknown, so the variant is - // not filtered out, and above all the resolution does not fail: a - // network hiccup must never be able to break an install. + It("completes the install on the entry's own payload when no probe answers", func() { + // A failed probe reports 0. That is an unknown, so nothing is filtered + // out and above all the resolution does not fail: a network hiccup + // must never be able to break an install. It must not be able to + // change what gets installed either, so an unmeasurable variant does + // not displace the payload the entry itself ships. resolved, variant, err := gallery.ResolveVariant(models, base, probing(gib(24), map[string]uint64{}), "") Expect(err).ToNot(HaveOccurred()) - Expect(variant.Model).To(Equal("qwen3-8b-vllm-awq")) - Expect(resolved.URL).To(Equal("file://vllm.yaml")) + Expect(variant.Model).To(Equal("qwen3-8b-gguf-q4")) + Expect(resolved.URL).To(Equal("file://gguf.yaml")) }) It("still installs the base when every probe fails and nothing else survives", func() { @@ -317,16 +320,27 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { var tempdir string var galleries []config.Gallery var systemState *system.SystemState + // galleryRevision keeps every gallery this suite writes distinct from every + // other, so the process-wide gallery cache cannot serve one spec's catalog + // to another. + galleryRevision := 0 // Every entry is described with an inline config_file rather than a URL so // the whole install runs off the local filesystem with no network access. + // + // Each call writes a fresh path and a fresh gallery name because the gallery + // listing is cached on the name and URL pair. A spec that edits its gallery + // mid-flight would otherwise keep reading the version it started with, and + // would silently prove nothing. newGallery := func(entries ...gallery.GalleryModel) { out, err := yaml.Marshal(entries) Expect(err).ToNot(HaveOccurred()) - galleryPath := filepath.Join(tempdir, "gallery.yaml") + name := fmt.Sprintf("test-%d", galleryRevision) + galleryRevision++ + galleryPath := filepath.Join(tempdir, name+".yaml") Expect(os.WriteFile(galleryPath, out, 0600)).To(Succeed()) - galleries = []config.Gallery{{Name: "test", URL: "file://" + galleryPath}} + galleries = []config.Gallery{{Name: name, URL: "file://" + galleryPath}} } entry := func(name, backend string) gallery.GalleryModel { @@ -352,7 +366,7 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { // entry's own, so only the entry-name overlay can keep the record under the // name the user asked for. The inline config_file branch seeds the name from // the already-renamed resolved entry and so cannot observe the overlay. - urlEntry := func(name, backend string) gallery.GalleryModel { + urlEntry := func(name, backend, size string) gallery.GalleryModel { payload := gallery.ModelConfig{ Name: name, Description: "entry " + name, @@ -367,6 +381,7 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { m.Name = name m.Description = "entry " + name m.URL = "file://" + payloadPath + m.Size = size return m } @@ -484,10 +499,13 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { // Without the entry-name overlay the record persists as "qwen3-8b-q8", // and the stable name the entry exists to provide is lost the moment // anything reads the record back. + // The declared sizes make the upgrade the largest build that fits, so + // resolution lands on the variant and the record it writes is the one + // under test here. newGallery( - withVariants(urlEntry("qwen3-8b-q4", "base-backend"), + withVariants(urlEntry("qwen3-8b-q4", "base-backend", "16MiB"), gallery.Variant{Model: "qwen3-8b-q8"}), - urlEntry("qwen3-8b-q8", "upgrade-backend"), + urlEntry("qwen3-8b-q8", "upgrade-backend", "256MiB"), ) Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) @@ -533,10 +551,14 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { }) It("records a pin and honors it on a plain reinstall", func() { + // The upgrade declares a size and the base does not, so auto-selection + // genuinely prefers the upgrade here. Without that the second install + // below would land on the base whether or not the pin was recalled, and + // the spec would prove nothing. newGallery( withVariants(entry("qwen3-8b-q4", "base-backend"), gallery.Variant{Model: "qwen3-8b-q8"}), - entry("qwen3-8b-q8", "upgrade-backend"), + sizedEntry("qwen3-8b-q8", "upgrade-backend", "16MiB"), ) Expect(install("qwen3-8b-q4", gallery.GalleryModel{}, gallery.WithVariant("qwen3-8b-q4"))).To(Succeed()) @@ -558,10 +580,12 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { }) It("honors a pin recorded under a custom install name", func() { + // Sized as above so auto-selection would take the upgrade, which is what + // makes the recall observable in the second install. newGallery( withVariants(entry("qwen3-8b-q4", "base-backend"), gallery.Variant{Model: "qwen3-8b-q8"}), - entry("qwen3-8b-q8", "upgrade-backend"), + sizedEntry("qwen3-8b-q8", "upgrade-backend", "16MiB"), ) req := gallery.GalleryModel{} @@ -581,6 +605,57 @@ var _ = Describe("InstallModelFromGallery with variant entries", func() { Expect(installedBackend("prod-llm")).To(Equal("base-backend")) }) + It("re-selects automatically when a recorded pin no longer exists in the gallery", func() { + // A pin is recorded, then the gallery is edited to rename the build it + // named. Without dropping the stale pin every later reinstall and every + // upgrade of this model fails forever, and the only remedy is deleting a + // dotfile the user has never heard of. + newGallery( + withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-q8"}), + sizedEntry("qwen3-8b-q8", "upgrade-backend", "16MiB"), + ) + Expect(install("qwen3-8b-q4", gallery.GalleryModel{}, gallery.WithVariant("qwen3-8b-q8"))).To(Succeed()) + + record, err := gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(record.PinnedVariant).To(Equal("qwen3-8b-q8")) + + // The gallery edit: the same build under a new name. + newGallery( + withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-q8-v2"}), + sizedEntry("qwen3-8b-q8-v2", "renamed-backend", "16MiB"), + ) + + Expect(install("qwen3-8b-q4", gallery.GalleryModel{})).To(Succeed()) + // Auto-selection ran: the renamed build is the largest that fits. + Expect(installedBackend("qwen3-8b-q4")).To(Equal("renamed-backend")) + + // The stale pin is cleared rather than carried forward, so the next + // install does not have to rediscover that it is unusable. + record, err = gallery.GetLocalModelConfiguration(tempdir, "qwen3-8b-q4") + Expect(err).ToNot(HaveOccurred()) + Expect(record.PinnedVariant).To(BeEmpty()) + Expect(record.ResolvedVariant).To(Equal("qwen3-8b-q8-v2")) + }) + + It("still refuses a variant the caller names on this request, recorded pin or not", func() { + // The caller-supplied pin and the recalled one must fail differently. + // This one is a request that cannot be honored, so reporting success + // for it would make a typo indistinguishable from a working choice. + newGallery( + withVariants(entry("qwen3-8b-q4", "base-backend"), + gallery.Variant{Model: "qwen3-8b-q8"}), + sizedEntry("qwen3-8b-q8", "upgrade-backend", "16MiB"), + ) + Expect(install("qwen3-8b-q4", gallery.GalleryModel{}, gallery.WithVariant("qwen3-8b-q8"))).To(Succeed()) + + err := install("qwen3-8b-q4", gallery.GalleryModel{}, gallery.WithVariant("qwen3-8b-q6")) + Expect(err).To(MatchError(gallery.ErrPinNotFound)) + Expect(err.Error()).To(ContainSubstring("qwen3-8b-q6")) + }) + It("writes each declared url only once into the persisted gallery file", func() { base := withVariants(entry("qwen3-8b-q4", "base-backend"), gallery.Variant{Model: "qwen3-8b-q8"}) diff --git a/core/services/galleryop/model_variant_test.go b/core/services/galleryop/model_variant_test.go index 689a2ecc442d..abbe903acbd2 100644 --- a/core/services/galleryop/model_variant_test.go +++ b/core/services/galleryop/model_variant_test.go @@ -62,13 +62,16 @@ var _ = Describe("LocalModelManager variant selection", func() { Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { Expect(os.RemoveAll(modelsDir)).To(Succeed()) }) - // The upgrade declares no size and carries no weight files, so the probe - // reports an unknown, the variant survives the filter and auto-selection - // would take it on any machine. Only an honored pin can land the install - // on the base. + // The upgrade declares a size no machine fails to clear and the base + // declares none, so the upgrade is the only measured fit and + // auto-selection takes it everywhere. Only an honored pin can land the + // install on the base. The size is read from the entry itself, so + // nothing here reaches the network. + upgrade := entry("qwen3-8b-q8", "upgrade-backend") + upgrade.Size = "16MiB" entries := []gallery.GalleryModel{ entry("qwen3-8b-q4", "base-backend", gallery.Variant{Model: "qwen3-8b-q8"}), - entry("qwen3-8b-q8", "upgrade-backend"), + upgrade, } out, err := yaml.Marshal(entries) Expect(err).ToNot(HaveOccurred()) diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index 68b08a0cf08d..1c66967f2f9f 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -160,10 +160,19 @@ carries a `variants` list, and installing it normally lets LocalAI choose: - variants whose backend cannot run on this machine are dropped; - variants that do not fit the available memory (VRAM on a GPU host, otherwise system RAM) are dropped; -- the largest remaining variant wins, because a bigger footprint means a higher +- the entry's own build is never dropped. It competes with whatever survived + rather than waiting for everything else to fail, so an entry that is itself + the largest build that fits keeps its own payload; +- the largest remaining build wins, because a bigger footprint means a higher quality build of the same model; -- if nothing survives, the entry's own build is installed. The entry is always - installable, on any machine. +- a build whose size could not be measured ranks below the entry's own build, + so an unreadable size never quietly displaces the payload the entry ships; +- if nothing else survives, the entry's own build is installed. The entry is + always installable, on any machine. + +Because the entry's own build competes on size like every other candidate, the +order of the list means nothing and a `variants` list may offer smaller builds, +larger ones, or both. Sizes are measured from the model's weights rather than downloaded, and cached. @@ -181,15 +190,16 @@ curl http://localhost:8080/api/models | jq '.models[] | select(.variants) | "auto_variant": "nanbeige4.1-3b-q8", "variants": [ { "model": "nanbeige4.1-3b-q8", "backend": "llama-cpp", "memory_bytes": 4187593113, "fits": true, "is_base": false }, - { "model": "nanbeige4.1-3b-q4", "backend": "llama-cpp", "memory_bytes": 0, "fits": true, "is_base": true } + { "model": "nanbeige4.1-3b-q4", "backend": "llama-cpp", "fits": true, "is_base": true } ] } ``` `auto_variant` is what installing without a choice would pick right now. `fits` -is whether auto-selection would consider that variant on this machine, and a -`memory_bytes` of 0 means the size could not be determined, not that the build -is free. `is_base` marks the entry's own build. +is whether auto-selection would consider that variant on this machine, and +`is_base` marks the entry's own build. `memory_bytes` is omitted entirely, as on +the second entry above, when the size could not be measured; read a missing +`memory_bytes` as unknown rather than as a free build. To install a specific one, pass its name as `variant`: diff --git a/pkg/mcp/localaitools/prompts/skills/install_chat_model.md b/pkg/mcp/localaitools/prompts/skills/install_chat_model.md index 54d2569d1cef..08fc4822bbba 100644 --- a/pkg/mcp/localaitools/prompts/skills/install_chat_model.md +++ b/pkg/mcp/localaitools/prompts/skills/install_chat_model.md @@ -6,7 +6,7 @@ Use this when the user wants to install a chat-capable model — including the c 2. Show the top results as a numbered list with name, gallery, short description, and license. If none match, say so and ask whether to broaden the search. 3. Wait for the user to pick. 4. Summarise the chosen install ("I'll install **`/`** — confirm?") and wait for confirmation. -5. On confirmation, call `install_model` with `gallery_name` and `model_name` from the chosen hit. Leave `variant` empty: LocalAI then auto-selects the largest build this machine's engines and free memory can actually run. Only set `variant` when the user names a specific build (e.g. "the Q8 one"), and pass the name exactly as the entry lists it — an unknown name fails the install rather than falling back to auto-selection. +5. On confirmation, call `install_model` with `gallery_name` and `model_name` from the chosen hit. Leave `variant` empty: LocalAI then auto-selects the largest build this machine's engines and free memory can actually run. Only set `variant` when the user names a specific build (e.g. "the Q8 one"), and pass the name exactly as the entry lists it: an unknown name fails the install rather than falling back to auto-selection. 6. Poll `get_job_status` with the returned job id. Report meaningful progress changes (every ~10–20%, plus completion). 7. When the job reports `processed: true` and no error, call `reload_models`, then `list_installed_models` with `capability: "chat"` to confirm the model is now visible. 8. Tell the user the model is ready and how to use it (its name as the `model` field in chat completions). diff --git a/pkg/system/capabilities.go b/pkg/system/capabilities.go index 9d0d3c3c452a..0c8fc25ab742 100644 --- a/pkg/system/capabilities.go +++ b/pkg/system/capabilities.go @@ -212,8 +212,10 @@ func (s *SystemState) BackendPreferenceTokens() []string { // // Why this exists alongside Capability: Capability resolves against a caller // supplied map and falls back to "default" then "cpu" when the detected value -// is absent from that map. Callers that express fallback themselves, such as -// model meta entries ordering their own candidates, need the undecorated value. +// is absent from that map, so its answer describes what that caller can serve +// rather than what the hardware is. A caller reasoning about the hardware +// itself, or reporting it to a human, cannot tell a genuinely detected +// "default" apart from a substituted one and needs the undecorated value. func (s *SystemState) DetectedCapability() string { return s.getSystemCapabilities() } From 7fd90419f5614b8602d20e5c0dd0307178e07523 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 19 Jul 2026 03:21:00 +0000 Subject: [PATCH 21/48] fix(gallery): budget variant memory from RAM when a GPU reports no VRAM Variant selection read its memory budget from VRAM whenever a GPU capability was detected, and from system RAM only when none was. Apple Silicon satisfies the first branch and fails the premise: arm64 macs report the metal capability unconditionally, without probing anything, while TotalAvailableVRAM has no discrete VRAM pool to find and returns zero. The budget therefore came out as zero on every Mac. Zero drops every variant carrying a known size, so the base build was installed on all of them however much memory the machine had. The feature was inert on the platform, and silently: falling back to the base is a legitimate outcome, so nothing looked wrong. Take VRAM only when it is actually a number, and fall back to RAM otherwise. On a unified-memory host RAM is not an approximation of the budget, it is the budget, since the GPU shares it. A discrete GPU whose VRAM could not be read also lands on RAM, which overstates what the card holds but understates nothing the host has; the previous zero understated both. An unreadable RAM figure still yields zero and still installs the base, so a genuinely unknown host is not talked into a larger download. This is what turned tests-apple red: "installs a fitting variant's payload under the entry's own name" asserts on selection, and the runner resolved to the base because its budget was zero. The added specs pin the branch directly rather than relying on a macOS runner to notice again. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- .../gallery/available_memory_internal_test.go | 73 +++++++++++++++++++ core/gallery/models.go | 11 ++- docs/content/features/model-gallery.md | 6 +- 3 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 core/gallery/available_memory_internal_test.go diff --git a/core/gallery/available_memory_internal_test.go b/core/gallery/available_memory_internal_test.go new file mode 100644 index 000000000000..e62b13800991 --- /dev/null +++ b/core/gallery/available_memory_internal_test.go @@ -0,0 +1,73 @@ +package gallery + +import ( + "os" + + "github.com/mudler/LocalAI/pkg/system" + "github.com/mudler/LocalAI/pkg/xsysinfo" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("availableModelMemory", func() { + // hostRAM is the figure the fallback resolves to. A host that cannot report + // its own RAM cannot exercise the fallback at all, so those specs skip + // rather than assert on a number that is legitimately unavailable. + hostRAM := func() uint64 { + ram, err := xsysinfo.GetSystemRAMInfo() + if err != nil || ram == nil || ram.Total == 0 { + Skip("this host does not report system RAM; the RAM fallback cannot be exercised here") + } + return ram.Total + } + + // stateWithCapability forces DetectedCapability for one spec. The state is + // built fresh so nothing is served from the capability cache. + stateWithCapability := func(capability string, vram uint64) *system.SystemState { + previous, had := os.LookupEnv("LOCALAI_FORCE_META_BACKEND_CAPABILITY") + Expect(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", capability)).To(Succeed()) + DeferCleanup(func() { + if had { + Expect(os.Setenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY", previous)).To(Succeed()) + return + } + Expect(os.Unsetenv("LOCALAI_FORCE_META_BACKEND_CAPABILITY")).To(Succeed()) + }) + + s := &system.SystemState{} + s.VRAM = vram + Expect(s.DetectedCapability()).To(Equal(capability), "the capability override did not take") + return s + } + + // A unified-memory host reports a GPU capability but no discrete VRAM pool. + // Apple Silicon is the case that matters: metal is reported unconditionally + // on arm64 macs while TotalAvailableVRAM finds nothing to add up. Reading + // that zero as the budget drops every sized variant and strands the host on + // the base build however much memory it has, which is what turned the macOS + // runner red. + It("falls back to system RAM when a detected GPU reports no VRAM", func() { + ram := hostRAM() + + got := availableModelMemory(stateWithCapability("metal", 0)) + + Expect(got).ToNot(BeZero(), "a unified-memory host was given a zero budget; every sized variant would be dropped") + Expect(got).To(Equal(ram)) + }) + + // A discrete GPU reports real VRAM, and that stays the budget: the model + // lives on the card, so system RAM is not what bounds it. + It("prefers VRAM when the GPU reports it", func() { + const vram = uint64(24) << 30 + + Expect(availableModelMemory(stateWithCapability("nvidia-cuda-12", vram))).To(Equal(vram)) + }) + + // With no GPU at all the model lives in system RAM. + It("uses system RAM when no GPU is detected", func() { + ram := hostRAM() + + Expect(availableModelMemory(stateWithCapability("default", 0))).To(Equal(ram)) + }) +}) diff --git a/core/gallery/models.go b/core/gallery/models.go index 18a314dfca40..726fe07b8522 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -287,16 +287,23 @@ const noGPUDetected = "default" // availableModelMemory reports how much memory a model may occupy on this host. // -// With a usable GPU the model lives in VRAM. Without one it lives in system +// With a discrete GPU the model lives in VRAM. Otherwise it lives in system // RAM, read through xsysinfo because that path is cgroup-aware: under // Kubernetes the container's limit, not the node's physical RAM, is what the // model actually gets. // +// A detected GPU whose VRAM reads as zero falls back to RAM rather than to +// zero. That is the normal shape of a unified-memory host, not an error: +// arm64 macs report the metal capability unconditionally, yet have no discrete +// VRAM pool for TotalAvailableVRAM to find, so the model's real budget is +// system RAM. Returning the zero here would strand every Mac on the base +// build no matter how much memory it has. +// // An unreadable RAM figure yields 0, which drops every variant with a known // requirement and installs the base. That is the safe direction: an unknown // host should not be talked into a larger download. func availableModelMemory(systemState *system.SystemState) uint64 { - if systemState.DetectedCapability() != noGPUDetected { + if systemState.DetectedCapability() != noGPUDetected && systemState.VRAM > 0 { return systemState.VRAM } ram, err := xsysinfo.GetSystemRAMInfo() diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index 1c66967f2f9f..b02ae336d088 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -158,8 +158,10 @@ quantizations, or the same weights served by a different engine. Such an entry carries a `variants` list, and installing it normally lets LocalAI choose: - variants whose backend cannot run on this machine are dropped; -- variants that do not fit the available memory (VRAM on a GPU host, otherwise - system RAM) are dropped; +- variants that do not fit the available memory are dropped. That budget is + VRAM on a discrete-GPU host, and system RAM otherwise — including on + unified-memory machines such as Apple Silicon, where the GPU shares system + RAM and reports no separate VRAM pool; - the entry's own build is never dropped. It competes with whatever survived rather than waiting for everything else to fail, so an entry that is itself the largest build that fits keeps its own payload; From 280c1d17b0c7206b8bc567c525aa54a960bc1901 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 19 Jul 2026 07:12:03 +0000 Subject: [PATCH 22/48] feat(ui): add a model variant picker to the models gallery PR #10943 shipped the server side: a gallery entry may declare `variants:`, `GET /api/models` attaches `variants` and `auto_variant` to declaring entries, and `POST /api/models/install/:id` accepts a `variant` query parameter. Nothing in the UI consumed any of it, so the feature was not reachable from the browser. This wires it up. modelsApi.install takes an optional second argument and appends an encoded `?variant=` only when one is given, so every existing call site keeps sending exactly the request it sent before. On the models table, an entry that declares variants gets a split button. The primary Install still installs the auto-selected build, because auto is the default and the point of the feature; the chevron opens a menu for a deliberate override. It follows the Backends.jsx precedent: one shared Popover re-anchored per row, rendering .action-menu items, which brings Escape, outside-click and focus return along with it. An entry that declares no variants renders exactly as it did before. A variant that does not fit is dimmed but stays selectable, since the server honors an explicit choice with a warning rather than refusing it. memory_bytes is omitempty on the wire, so an absent key means the size is unknown and never zero. A single helper guards both the menu and the detail row, because formatBytes would otherwise render a falsy value as "0 B", which reads as "needs nothing". The expanded detail row gains a Variants section listing each build's backend, size, whether it fits, which is the entry's own build, and which one auto-selection would pick, built from the existing DetailRow helper and .badge classes. Eight Playwright specs cover the picker, including that plain Install sends no variant parameter and that choosing one sends it. One pre-existing assertion was scoped with .first(): the Variants section legitimately adds more llama-cpp badges to the detail row, which tripped strict mode. UI line coverage 49.42% -> 49.36% against a 40.0 baseline and 0.8pp tolerance; branch coverage rose 72.04% -> 72.66%. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/http/react-ui/e2e/models-gallery.spec.js | 154 +++++++++++++++++- .../react-ui/public/locales/de/models.json | 11 ++ .../react-ui/public/locales/en/models.json | 11 ++ .../react-ui/public/locales/es/models.json | 11 ++ .../react-ui/public/locales/id/models.json | 11 ++ .../react-ui/public/locales/it/models.json | 11 ++ .../react-ui/public/locales/zh-CN/models.json | 11 ++ core/http/react-ui/src/pages/Models.jsx | 114 ++++++++++++- core/http/react-ui/src/utils/api.js | 9 +- 9 files changed, 338 insertions(+), 5 deletions(-) diff --git a/core/http/react-ui/e2e/models-gallery.spec.js b/core/http/react-ui/e2e/models-gallery.spec.js index d45b5eedac99..fdf107bfdc9b 100644 --- a/core/http/react-ui/e2e/models-gallery.spec.js +++ b/core/http/react-ui/e2e/models-gallery.spec.js @@ -8,6 +8,38 @@ const MOCK_MODELS_RESPONSE = { backend: "llama-cpp", installed: false, tags: ["chat"], + // memory_bytes is omitempty server-side, so the mlx variant deliberately + // carries no key at all: the UI must render that as unknown, never 0 B. + variants: [ + { + model: "llama-model", + backend: "llama-cpp", + memory_bytes: 4 * 1024 * 1024 * 1024, + fits: true, + is_base: true, + }, + { + model: "llama-model-q8", + backend: "llama-cpp", + memory_bytes: 8 * 1024 * 1024 * 1024, + fits: true, + is_base: false, + }, + { + model: "llama-model-mlx", + backend: "mlx", + fits: true, + is_base: false, + }, + { + model: "llama-model-f16", + backend: "llama-cpp", + memory_bytes: 40 * 1024 * 1024 * 1024, + fits: false, + is_base: false, + }, + ], + auto_variant: "llama-model-q8", }, { name: "whisper-model", @@ -173,7 +205,9 @@ test.describe("Models Gallery - Backend Features", () => { // The detail view should show Backend label and value const detail = page.locator('td[colspan="8"]'); await expect(detail.locator("text=Backend")).toBeVisible(); - await expect(detail.locator("text=llama-cpp")).toBeVisible(); + // The Backend DetailRow renders before the Variants section, which lists a + // per-variant backend badge of its own, so scope to the first match. + await expect(detail.locator("text=llama-cpp").first()).toBeVisible(); }); }); @@ -433,3 +467,121 @@ test.describe("Models Gallery - Empty State", () => { await expect(page.locator("tr", { hasText: "llama-model" })).toBeVisible(); }); }); + +test.describe("Models Gallery - Variant picker", () => { + // installUrls records every install request so a test can assert both the + // presence and the absence of the ?variant= parameter. + let installUrls; + + test.beforeEach(async ({ page }) => { + installUrls = []; + await page.route("**/api/models*", (route) => { + route.fulfill({ + contentType: "application/json", + body: JSON.stringify(MOCK_MODELS_RESPONSE), + }); + }); + await page.route("**/api/models/install/**", (route) => { + installUrls.push(route.request().url()); + return route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ jobID: "variant-install" }), + }); + }); + await page.goto("/app/models"); + await expect(page.locator("th", { hasText: "Backend" })).toBeVisible({ + timeout: 10_000, + }); + }); + + const variantRow = (page) => page.locator("tr", { hasText: "llama-model" }).first(); + const plainRow = (page) => + page.locator("tr", { hasText: "stablediffusion-model" }).first(); + + test("an entry that declares variants shows the split-button chevron", async ({ + page, + }) => { + await expect( + variantRow(page).getByRole("button", { name: "Choose a variant" }), + ).toBeVisible(); + }); + + test("an entry without variants renders no chevron", async ({ page }) => { + await expect( + plainRow(page).getByRole("button", { name: "Choose a variant" }), + ).toHaveCount(0); + // and still offers an ordinary install + await expect( + plainRow(page).locator("button.btn-primary"), + ).toHaveCount(1); + }); + + test("plain Install sends no variant parameter", async ({ page }) => { + await plainRow(page).locator("button.btn-primary").click(); + await expect.poll(() => installUrls.length).toBe(1); + expect(installUrls[0]).not.toContain("variant="); + }); + + test("the auto-selected variant is marked in the menu", async ({ page }) => { + await variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + const menu = page.locator(".action-menu"); + await expect(menu).toBeVisible(); + const autoItem = menu.locator(".action-menu__item", { + hasText: "llama-model-q8", + }); + await expect(autoItem.locator(".badge", { hasText: "Auto" })).toBeVisible(); + // the base build is identifiable too + await expect( + menu + .locator(".action-menu__item", { hasText: "llama-model" }) + .first() + .locator(".badge", { hasText: "Base build" }), + ).toBeVisible(); + }); + + test("a variant with no memory_bytes renders as unknown, not 0", async ({ + page, + }) => { + await variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + const mlxItem = page.locator(".action-menu__item", { + hasText: "llama-model-mlx", + }); + await expect(mlxItem).toContainText("Unknown size"); + await expect(mlxItem).not.toContainText("0 B"); + }); + + test("a variant that does not fit is still selectable", async ({ page }) => { + await variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + const f16 = page.locator(".action-menu__item", { + hasText: "llama-model-f16", + }); + await expect(f16.locator(".badge", { hasText: "Does not fit" })).toBeVisible(); + await expect(f16).toBeEnabled(); + }); + + test("choosing a specific variant sends ?variant= on the install", async ({ + page, + }) => { + await variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + await page + .locator(".action-menu__item", { hasText: "llama-model-mlx" }) + .click(); + await expect.poll(() => installUrls.length).toBe(1); + expect(installUrls[0]).toContain("variant=llama-model-mlx"); + }); + + test("the expanded detail row lists every variant", async ({ page }) => { + await variantRow(page).click(); + const detail = page.locator('td[colspan="8"]'); + await expect(detail).toContainText("Variants"); + await expect(detail).toContainText("llama-model-q8"); + await expect(detail).toContainText("llama-model-mlx"); + await expect(detail).toContainText("llama-model-f16"); + await expect(detail).toContainText("Unknown size"); + await expect(detail).toContainText("Auto-selected"); + await expect(detail).toContainText("Base build"); + await expect(detail).toContainText("Does not fit"); + await expect(detail).toContainText("mlx"); + }); +}); diff --git a/core/http/react-ui/public/locales/de/models.json b/core/http/react-ui/public/locales/de/models.json index e5929d832e31..cf99a6567e91 100644 --- a/core/http/react-ui/public/locales/de/models.json +++ b/core/http/react-ui/public/locales/de/models.json @@ -83,5 +83,16 @@ "loadFailed": "Laden der Modelle fehlgeschlagen: {{message}}", "installFailed": "Installation fehlgeschlagen: {{message}}", "deleteFailed": "Löschen fehlgeschlagen: {{message}}" + }, + "variants": { + "title": "Variants", + "chooseVariant": "Choose a variant", + "auto": "Auto", + "autoSelected": "Auto-selected", + "base": "Base build", + "fits": "Fits", + "doesNotFit": "Does not fit", + "unknownSize": "Unknown size", + "unknownBackend": "Unknown backend" } } diff --git a/core/http/react-ui/public/locales/en/models.json b/core/http/react-ui/public/locales/en/models.json index bd23d389ee57..ac2e8b71ef2d 100644 --- a/core/http/react-ui/public/locales/en/models.json +++ b/core/http/react-ui/public/locales/en/models.json @@ -108,5 +108,16 @@ "selectModel": "Select model...", "searchPlaceholder": "Search models...", "noModels": "No models available" + }, + "variants": { + "title": "Variants", + "chooseVariant": "Choose a variant", + "auto": "Auto", + "autoSelected": "Auto-selected", + "base": "Base build", + "fits": "Fits", + "doesNotFit": "Does not fit", + "unknownSize": "Unknown size", + "unknownBackend": "Unknown backend" } } diff --git a/core/http/react-ui/public/locales/es/models.json b/core/http/react-ui/public/locales/es/models.json index ac4275fbbbe4..578f5b71c744 100644 --- a/core/http/react-ui/public/locales/es/models.json +++ b/core/http/react-ui/public/locales/es/models.json @@ -83,5 +83,16 @@ "loadFailed": "Error al cargar modelos: {{message}}", "installFailed": "Error al instalar: {{message}}", "deleteFailed": "Error al eliminar: {{message}}" + }, + "variants": { + "title": "Variants", + "chooseVariant": "Choose a variant", + "auto": "Auto", + "autoSelected": "Auto-selected", + "base": "Base build", + "fits": "Fits", + "doesNotFit": "Does not fit", + "unknownSize": "Unknown size", + "unknownBackend": "Unknown backend" } } diff --git a/core/http/react-ui/public/locales/id/models.json b/core/http/react-ui/public/locales/id/models.json index a8c5404fa43d..f66622f586f4 100644 --- a/core/http/react-ui/public/locales/id/models.json +++ b/core/http/react-ui/public/locales/id/models.json @@ -96,5 +96,16 @@ "selectModel": "Pilih model...", "searchPlaceholder": "Cari model...", "noModels": "Model tidak tersedia" + }, + "variants": { + "title": "Variants", + "chooseVariant": "Choose a variant", + "auto": "Auto", + "autoSelected": "Auto-selected", + "base": "Base build", + "fits": "Fits", + "doesNotFit": "Does not fit", + "unknownSize": "Unknown size", + "unknownBackend": "Unknown backend" } } diff --git a/core/http/react-ui/public/locales/it/models.json b/core/http/react-ui/public/locales/it/models.json index 25c371eb1be6..6f963e514253 100644 --- a/core/http/react-ui/public/locales/it/models.json +++ b/core/http/react-ui/public/locales/it/models.json @@ -83,5 +83,16 @@ "loadFailed": "Caricamento modelli fallito: {{message}}", "installFailed": "Installazione fallita: {{message}}", "deleteFailed": "Eliminazione fallita: {{message}}" + }, + "variants": { + "title": "Variants", + "chooseVariant": "Choose a variant", + "auto": "Auto", + "autoSelected": "Auto-selected", + "base": "Base build", + "fits": "Fits", + "doesNotFit": "Does not fit", + "unknownSize": "Unknown size", + "unknownBackend": "Unknown backend" } } diff --git a/core/http/react-ui/public/locales/zh-CN/models.json b/core/http/react-ui/public/locales/zh-CN/models.json index 246153e6894e..d25e608aff2a 100644 --- a/core/http/react-ui/public/locales/zh-CN/models.json +++ b/core/http/react-ui/public/locales/zh-CN/models.json @@ -83,5 +83,16 @@ "loadFailed": "加载模型失败：{{message}}", "installFailed": "安装失败：{{message}}", "deleteFailed": "删除失败：{{message}}" + }, + "variants": { + "title": "Variants", + "chooseVariant": "Choose a variant", + "auto": "Auto", + "autoSelected": "Auto-selected", + "base": "Base build", + "fits": "Fits", + "doesNotFit": "Does not fit", + "unknownSize": "Unknown size", + "unknownBackend": "Unknown backend" } } diff --git a/core/http/react-ui/src/pages/Models.jsx b/core/http/react-ui/src/pages/Models.jsx index cf10fdfb58f7..ee5585d433e9 100644 --- a/core/http/react-ui/src/pages/Models.jsx +++ b/core/http/react-ui/src/pages/Models.jsx @@ -1,4 +1,4 @@ -import { useState, useCallback, useEffect } from 'react' +import { useState, useCallback, useEffect, useRef } from 'react' import { useNavigate, useOutletContext, useLocation } from 'react-router-dom' import { useTranslation } from 'react-i18next' import { fromState } from '../utils/editorNav' @@ -14,6 +14,8 @@ import GalleryLoader from '../components/GalleryLoader' import Toggle from '../components/Toggle' import ResponsiveTable from '../components/ResponsiveTable' import RecommendedModels from '../components/RecommendedModels' +import Popover from '../components/Popover' +import { formatBytes } from '../utils/format' import React from 'react' @@ -68,6 +70,10 @@ export default function Models() { const [estimates, setEstimates] = useState({}) const [contextSize, setContextSize] = useState(CONTEXT_SIZES[0]) const [confirmDialog, setConfirmDialog] = useState(null) + // Index of the row whose variant split-menu is open, or null. A single + // Popover is re-anchored per row rather than one instance per row. + const [variantMenuFor, setVariantMenuFor] = useState(null) + const variantMenuAnchorRef = useRef(null) const [fitsFilter, setFitsFilter] = useState(() => { try { return localStorage.getItem(FITS_FILTER_STORAGE_KEY) === '1' @@ -187,10 +193,10 @@ export default function Models() { } } - const handleInstall = async (modelId) => { + const handleInstall = async (modelId, variant) => { try { setInstalling(prev => new Map(prev).set(modelId, Date.now())) - await modelsApi.install(modelId) + await modelsApi.install(modelId, variant) } catch (err) { addToast(t('errors.installFailed', { message: err.message }), 'error') } @@ -423,6 +429,7 @@ export default function Models() { const progress = getOperationProgress(name) const fit = fitsGpu(vramBytes) const isExpanded = expandedRow === idx + const hasVariants = model.variants?.length > 0 return ( @@ -553,6 +560,34 @@ export default function Models() { + ) : hasVariants ? ( + // Split button: the primary keeps installing the + // auto-selected build, so the default path is + // unchanged. The chevron is the deliberate + // override. +
+ + +
) : ( + ) + })} + + ) } +// variantSizeLabel renders a variant footprint. memory_bytes is omitempty on +// the wire, so an absent key means the probe could not determine a size; it +// must never render as "0 B", which would read as "needs nothing". +function variantSizeLabel(variant, t) { + return variant?.memory_bytes ? formatBytes(variant.memory_bytes) : t('variants.unknownSize') +} + function DetailRow({ label, children }) { if (!children) return null return ( @@ -661,6 +747,28 @@ function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setE ) : null} + {model.variants?.length > 0 && ( + +
+ {model.variants.map(v => ( +
+ {v.model} + {v.backend || t('variants.unknownBackend')} + {variantSizeLabel(v, t)} + + {v.fits ? t('variants.fits') : t('variants.doesNotFit')} + + {v.is_base && {t('variants.base')}} + {v.model === model.auto_variant && ( + + {t('variants.autoSelected')} + + )} +
+ ))} +
+ + )} {model.license && {model.license}} diff --git a/core/http/react-ui/src/utils/api.js b/core/http/react-ui/src/utils/api.js index 33e200506df7..3895dae3c865 100644 --- a/core/http/react-ui/src/utils/api.js +++ b/core/http/react-ui/src/utils/api.js @@ -85,7 +85,14 @@ export const modelsApi = { listV1: () => fetchJSON(API_CONFIG.endpoints.modelsList), listCapabilities: () => fetchJSON(API_CONFIG.endpoints.modelsCapabilities), listAliases: () => fetchJSON(API_CONFIG.endpoints.modelsAliases), - install: (id) => postJSON(API_CONFIG.endpoints.installModel(id), {}), + // variant is optional. Omitting it lets the server auto-select the best + // build for this host, which is what the listing's auto_variant predicted. + install: (id, variant) => postJSON( + variant + ? `${API_CONFIG.endpoints.installModel(id)}?variant=${encodeURIComponent(variant)}` + : API_CONFIG.endpoints.installModel(id), + {} + ), delete: (id) => postJSON(API_CONFIG.endpoints.deleteModel(id), {}), estimate: (id, contexts) => fetchJSON( buildUrl(API_CONFIG.endpoints.modelEstimate(id), From 4f52c737ad53e2f473b29628e01a758057a7ab86 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 19 Jul 2026 07:29:23 +0000 Subject: [PATCH 23/48] fix(gallery): describe model variants from a companion endpoint Variant description probes each referenced entry's weight files over the network: an HTTP HEAD plus a ranged GET, serial, five seconds per probe with no aggregate deadline. Running it inline in GET /api/models made one listing cost (entries x variants) round trips. The Manage page fetches with items=9999, so at 200 declaring entries that is ~1000 serial probes, minutes of a blocked handler and gigabytes of range traffic for a single page load. Only one entry declares variants today, but the feature exists so that many will. Follow the precedent already set for VRAM estimates. The listing now reports only has_variants, a length check on loaded metadata that touches nothing, and GET /api/models/variants/:id returns the description for one entry, mirroring estimate/:id in route shape, auth and error handling. DescribeVariants itself is unchanged; only its caller moved. The picker fetches lazily at the two points where a user asks to see variants, opening the split-button menu and expanding the detail row, and caches per entry for the page session. An entry declaring no variants issues no request at all. A spec counts real HTTP hits on the weight files, so it goes red if description becomes reachable from the listing path again through any caller. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/http/react-ui/e2e/models-gallery.spec.js | 151 ++++++++++--- .../react-ui/public/locales/de/models.json | 3 +- .../react-ui/public/locales/en/models.json | 3 +- .../react-ui/public/locales/es/models.json | 3 +- .../react-ui/public/locales/id/models.json | 3 +- .../react-ui/public/locales/it/models.json | 3 +- .../react-ui/public/locales/zh-CN/models.json | 3 +- core/http/react-ui/src/pages/Models.jsx | 154 +++++++++---- core/http/react-ui/src/utils/api.js | 3 + core/http/react-ui/src/utils/config.js | 1 + core/http/routes/ui_api.go | 80 +++++-- core/http/routes/ui_api_variants_test.go | 211 ++++++++++++++++++ docs/content/features/model-gallery.md | 25 ++- 13 files changed, 523 insertions(+), 120 deletions(-) create mode 100644 core/http/routes/ui_api_variants_test.go diff --git a/core/http/react-ui/e2e/models-gallery.spec.js b/core/http/react-ui/e2e/models-gallery.spec.js index fdf107bfdc9b..5d3934710974 100644 --- a/core/http/react-ui/e2e/models-gallery.spec.js +++ b/core/http/react-ui/e2e/models-gallery.spec.js @@ -8,38 +8,10 @@ const MOCK_MODELS_RESPONSE = { backend: "llama-cpp", installed: false, tags: ["chat"], - // memory_bytes is omitempty server-side, so the mlx variant deliberately - // carries no key at all: the UI must render that as unknown, never 0 B. - variants: [ - { - model: "llama-model", - backend: "llama-cpp", - memory_bytes: 4 * 1024 * 1024 * 1024, - fits: true, - is_base: true, - }, - { - model: "llama-model-q8", - backend: "llama-cpp", - memory_bytes: 8 * 1024 * 1024 * 1024, - fits: true, - is_base: false, - }, - { - model: "llama-model-mlx", - backend: "mlx", - fits: true, - is_base: false, - }, - { - model: "llama-model-f16", - backend: "llama-cpp", - memory_bytes: 40 * 1024 * 1024 * 1024, - fits: false, - is_base: false, - }, - ], - auto_variant: "llama-model-q8", + // The listing carries only the declaration flag. Describing variants + // costs the server a network probe each, so the description lives + // behind /api/models/variants/:id and is fetched on demand. + has_variants: true, }, { name: "whisper-model", @@ -468,13 +440,57 @@ test.describe("Models Gallery - Empty State", () => { }); }); +// The variant description the companion endpoint returns for llama-model. +// memory_bytes is omitempty server-side, so the mlx variant deliberately +// carries no key at all: the UI must render that as unknown, never 0 B. +const MOCK_VARIANTS_RESPONSE = { + variants: [ + { + model: "llama-model", + backend: "llama-cpp", + memory_bytes: 4 * 1024 * 1024 * 1024, + fits: true, + is_base: true, + }, + { + model: "llama-model-q8", + backend: "llama-cpp", + memory_bytes: 8 * 1024 * 1024 * 1024, + fits: true, + is_base: false, + }, + { + model: "llama-model-mlx", + backend: "mlx", + fits: true, + is_base: false, + }, + { + model: "llama-model-f16", + backend: "llama-cpp", + memory_bytes: 40 * 1024 * 1024 * 1024, + fits: false, + is_base: false, + }, + ], + auto_selected: "llama-model-q8", +}; + test.describe("Models Gallery - Variant picker", () => { // installUrls records every install request so a test can assert both the // presence and the absence of the ?variant= parameter. let installUrls; + // variantUrls records every companion-endpoint request. It is what proves + // the description is fetched lazily and cached, rather than being paid for + // by every row on page load. + let variantUrls; + // Held requests let a test observe the in-flight state rather than racing it. + let releaseVariants; test.beforeEach(async ({ page }) => { installUrls = []; + variantUrls = []; + releaseVariants = null; await page.route("**/api/models*", (route) => { route.fulfill({ contentType: "application/json", @@ -489,6 +505,15 @@ test.describe("Models Gallery - Variant picker", () => { body: JSON.stringify({ jobID: "variant-install" }), }); }); + await page.route("**/api/models/variants/**", async (route) => { + variantUrls.push(route.request().url()); + if (releaseVariants) await releaseVariants; + return route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify(MOCK_VARIANTS_RESPONSE), + }); + }); await page.goto("/app/models"); await expect(page.locator("th", { hasText: "Backend" })).toBeVisible({ timeout: 10_000, @@ -498,6 +523,15 @@ test.describe("Models Gallery - Variant picker", () => { const variantRow = (page) => page.locator("tr", { hasText: "llama-model" }).first(); const plainRow = (page) => page.locator("tr", { hasText: "stablediffusion-model" }).first(); + const openMenu = (page) => + variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + + test("the listing alone fetches no variant descriptions", async ({ page }) => { + // The whole point of the companion endpoint: a page load costs zero + // probes no matter how many entries declare variants. + await expect(page.locator("tbody tr").first()).toBeVisible(); + expect(variantUrls).toHaveLength(0); + }); test("an entry that declares variants shows the split-button chevron", async ({ page, @@ -517,14 +551,57 @@ test.describe("Models Gallery - Variant picker", () => { ).toHaveCount(1); }); + test("an entry without variants fetches nothing even when expanded", async ({ + page, + }) => { + await plainRow(page).click(); + await expect(page.locator('td[colspan="8"]')).toBeVisible(); + expect(variantUrls).toHaveLength(0); + }); + test("plain Install sends no variant parameter", async ({ page }) => { await plainRow(page).locator("button.btn-primary").click(); await expect.poll(() => installUrls.length).toBe(1); expect(installUrls[0]).not.toContain("variant="); }); + test("opening the menu fetches the description once and caches it", async ({ + page, + }) => { + await openMenu(page); + await expect(page.locator(".action-menu")).toBeVisible(); + await expect.poll(() => variantUrls.length).toBe(1); + expect(variantUrls[0]).toContain("/api/models/variants/llama-model"); + + // Close and reopen: the cached answer must be reused. + await page.keyboard.press("Escape"); + await openMenu(page); + await expect( + page.locator(".action-menu__item", { hasText: "llama-model-q8" }), + ).toBeVisible(); + expect(variantUrls).toHaveLength(1); + }); + + test("the menu shows a loading state while the description is in flight", async ({ + page, + }) => { + let unblock; + releaseVariants = new Promise((resolve) => { + unblock = resolve; + }); + await openMenu(page); + await expect(page.locator(".action-menu")).toContainText("Loading variants"); + unblock(); + await expect( + page.locator(".action-menu__item", { hasText: "llama-model-q8" }), + ).toBeVisible(); + await expect(page.locator(".action-menu")).not.toContainText( + "Loading variants", + ); + }); + test("the auto-selected variant is marked in the menu", async ({ page }) => { - await variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + await openMenu(page); const menu = page.locator(".action-menu"); await expect(menu).toBeVisible(); const autoItem = menu.locator(".action-menu__item", { @@ -543,7 +620,7 @@ test.describe("Models Gallery - Variant picker", () => { test("a variant with no memory_bytes renders as unknown, not 0", async ({ page, }) => { - await variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + await openMenu(page); const mlxItem = page.locator(".action-menu__item", { hasText: "llama-model-mlx", }); @@ -552,7 +629,7 @@ test.describe("Models Gallery - Variant picker", () => { }); test("a variant that does not fit is still selectable", async ({ page }) => { - await variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + await openMenu(page); const f16 = page.locator(".action-menu__item", { hasText: "llama-model-f16", }); @@ -563,7 +640,7 @@ test.describe("Models Gallery - Variant picker", () => { test("choosing a specific variant sends ?variant= on the install", async ({ page, }) => { - await variantRow(page).getByRole("button", { name: "Choose a variant" }).click(); + await openMenu(page); await page .locator(".action-menu__item", { hasText: "llama-model-mlx" }) .click(); @@ -583,5 +660,7 @@ test.describe("Models Gallery - Variant picker", () => { await expect(detail).toContainText("Base build"); await expect(detail).toContainText("Does not fit"); await expect(detail).toContainText("mlx"); + // Expanding is the second trigger point, so it pays for exactly one fetch. + expect(variantUrls).toHaveLength(1); }); }); diff --git a/core/http/react-ui/public/locales/de/models.json b/core/http/react-ui/public/locales/de/models.json index cf99a6567e91..9142a1b8b624 100644 --- a/core/http/react-ui/public/locales/de/models.json +++ b/core/http/react-ui/public/locales/de/models.json @@ -93,6 +93,7 @@ "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", - "unknownBackend": "Unknown backend" + "unknownBackend": "Unknown backend", + "loading": "Loading variants..." } } diff --git a/core/http/react-ui/public/locales/en/models.json b/core/http/react-ui/public/locales/en/models.json index ac2e8b71ef2d..9c750774f119 100644 --- a/core/http/react-ui/public/locales/en/models.json +++ b/core/http/react-ui/public/locales/en/models.json @@ -118,6 +118,7 @@ "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", - "unknownBackend": "Unknown backend" + "unknownBackend": "Unknown backend", + "loading": "Loading variants..." } } diff --git a/core/http/react-ui/public/locales/es/models.json b/core/http/react-ui/public/locales/es/models.json index 578f5b71c744..69ca19daec99 100644 --- a/core/http/react-ui/public/locales/es/models.json +++ b/core/http/react-ui/public/locales/es/models.json @@ -93,6 +93,7 @@ "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", - "unknownBackend": "Unknown backend" + "unknownBackend": "Unknown backend", + "loading": "Loading variants..." } } diff --git a/core/http/react-ui/public/locales/id/models.json b/core/http/react-ui/public/locales/id/models.json index f66622f586f4..3acff3afb249 100644 --- a/core/http/react-ui/public/locales/id/models.json +++ b/core/http/react-ui/public/locales/id/models.json @@ -106,6 +106,7 @@ "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", - "unknownBackend": "Unknown backend" + "unknownBackend": "Unknown backend", + "loading": "Loading variants..." } } diff --git a/core/http/react-ui/public/locales/it/models.json b/core/http/react-ui/public/locales/it/models.json index 6f963e514253..15eca936fdc9 100644 --- a/core/http/react-ui/public/locales/it/models.json +++ b/core/http/react-ui/public/locales/it/models.json @@ -93,6 +93,7 @@ "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", - "unknownBackend": "Unknown backend" + "unknownBackend": "Unknown backend", + "loading": "Loading variants..." } } diff --git a/core/http/react-ui/public/locales/zh-CN/models.json b/core/http/react-ui/public/locales/zh-CN/models.json index d25e608aff2a..88cd49d7d998 100644 --- a/core/http/react-ui/public/locales/zh-CN/models.json +++ b/core/http/react-ui/public/locales/zh-CN/models.json @@ -93,6 +93,7 @@ "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", - "unknownBackend": "Unknown backend" + "unknownBackend": "Unknown backend", + "loading": "Loading variants..." } } diff --git a/core/http/react-ui/src/pages/Models.jsx b/core/http/react-ui/src/pages/Models.jsx index ee5585d433e9..75375632b15e 100644 --- a/core/http/react-ui/src/pages/Models.jsx +++ b/core/http/react-ui/src/pages/Models.jsx @@ -74,6 +74,11 @@ export default function Models() { // Popover is re-anchored per row rather than one instance per row. const [variantMenuFor, setVariantMenuFor] = useState(null) const variantMenuAnchorRef = useRef(null) + // Variant descriptions, keyed by model name. The listing only tells us + // whether an entry declares any; describing them costs the server a network + // probe per variant, so we ask for one entry at a time and keep the answer + // for the rest of the page session. + const [variantData, setVariantData] = useState({}) const [fitsFilter, setFitsFilter] = useState(() => { try { return localStorage.getItem(FITS_FILTER_STORAGE_KEY) === '1' @@ -193,6 +198,21 @@ export default function Models() { } } + // Fetches an entry's variant description once. Called from the two points + // where a user actually asks to see variants: opening the split-button menu + // and expanding the detail row. An entry that declares none never gets here, + // so it issues no request at all. + const loadVariants = useCallback((id) => { + if (!id) return + setVariantData(prev => { + if (prev[id]) return prev + modelsApi.variants(id) + .then(data => setVariantData(p => ({ ...p, [id]: { loading: false, ...data } }))) + .catch(() => setVariantData(p => ({ ...p, [id]: { loading: false, variants: [] } }))) + return { ...prev, [id]: { loading: true, variants: [] } } + }) + }, []) + const handleInstall = async (modelId, variant) => { try { setInstalling(prev => new Map(prev).set(modelId, Date.now())) @@ -429,12 +449,16 @@ export default function Models() { const progress = getOperationProgress(name) const fit = fitsGpu(vramBytes) const isExpanded = expandedRow === idx - const hasVariants = model.variants?.length > 0 + const hasVariants = !!model.has_variants return ( { setExpandedRow(isExpanded ? null : idx); setExpandedFiles(false) }} + onClick={() => { + if (!isExpanded && hasVariants) loadVariants(name) + setExpandedRow(isExpanded ? null : idx) + setExpandedFiles(false) + }} style={{ cursor: 'pointer' }} > {/* Chevron */} @@ -578,7 +602,10 @@ export default function Models() { - ) - })} - - + onChoose={handleInstall} + t={t} + /> ) } +// VariantMenu is the split-button dropdown. It is one instance re-anchored to +// whichever row is active, so Escape, outside-click and focus return come from +// Popover rather than being reimplemented per row. +function VariantMenu({ anchor, model, variantData, onClose, onChoose, t }) { + const name = model ? (model.name || model.id) : null + const data = name ? variantData[name] : null + return ( + +
+ {data?.loading && ( + // The description is a round trip, so the menu says so rather than + // opening empty and looking broken. +
+ + {t('variants.loading')} +
+ )} + {(data?.variants || []).map(v => { + const isAuto = v.model === data?.auto_selected + return ( + + ) + })} +
+ + ) +} + // variantSizeLabel renders a variant footprint. memory_bytes is omitempty on // the wire, so an absent key means the probe could not determine a size; it // must never render as "0 B", which would read as "needs nothing". @@ -707,7 +756,7 @@ function DetailRow({ label, children }) { ) } -function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setExpandedFiles, t }) { +function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setExpandedFiles, variantData, t }) { const files = model.additionalFiles || model.files || [] return (
@@ -747,10 +796,17 @@ function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setE ) : null} - {model.variants?.length > 0 && ( + {variantData?.loading && ( + + + {t('variants.loading')} + + + )} + {variantData?.variants?.length > 0 && (
- {model.variants.map(v => ( + {variantData.variants.map(v => (
{v.model} {v.backend || t('variants.unknownBackend')} @@ -759,7 +815,7 @@ function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setE {v.fits ? t('variants.fits') : t('variants.doesNotFit')} {v.is_base && {t('variants.base')}} - {v.model === model.auto_variant && ( + {v.model === variantData.auto_selected && ( {t('variants.autoSelected')} diff --git a/core/http/react-ui/src/utils/api.js b/core/http/react-ui/src/utils/api.js index 3895dae3c865..e2b76ba32383 100644 --- a/core/http/react-ui/src/utils/api.js +++ b/core/http/react-ui/src/utils/api.js @@ -98,6 +98,9 @@ export const modelsApi = { buildUrl(API_CONFIG.endpoints.modelEstimate(id), contexts?.length ? { contexts: contexts.join(',') } : {}) ), + // Companion to estimate: the listing reports only has_variants, so the + // description is fetched per entry, on demand. + variants: (id) => fetchJSON(API_CONFIG.endpoints.modelVariants(id)), getConfig: (id) => postJSON(API_CONFIG.endpoints.modelConfig(id), {}), getConfigJson: (name) => fetchJSON(API_CONFIG.endpoints.modelConfigJson(name)), getJob: (uid) => fetchJSON(API_CONFIG.endpoints.modelJob(uid)), diff --git a/core/http/react-ui/src/utils/config.js b/core/http/react-ui/src/utils/config.js index d1bd4d3351f0..3ddf19eb79b7 100644 --- a/core/http/react-ui/src/utils/config.js +++ b/core/http/react-ui/src/utils/config.js @@ -10,6 +10,7 @@ export const API_CONFIG = { installModel: (id) => `/api/models/install/${id}`, deleteModel: (id) => `/api/models/delete/${id}`, modelEstimate: (id) => `/api/models/estimate/${id}`, + modelVariants: (id) => `/api/models/variants/${id}`, modelConfig: (id) => `/api/models/config/${id}`, modelConfigJson: (name) => `/api/models/config-json/${name}`, configMetadata: '/api/models/config-metadata', diff --git a/core/http/routes/ui_api.go b/core/http/routes/ui_api.go index b30dd256179e..ec5f4258ea89 100644 --- a/core/http/routes/ui_api.go +++ b/core/http/routes/ui_api.go @@ -413,11 +413,6 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model }) } - // Kept before the filters and pagination below narrow `models`: a - // variant references another gallery entry by name, and that entry may - // well have been filtered off this page. - allModels := models - // Get all available tags allTags := map[string]struct{}{} tags := []string{} @@ -590,24 +585,18 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model "voice_cloning": m.VoiceCloningCapability(appConfig.SystemState.Model.ModelsPath), } - // Variants are described only for the entries that declare them. - // DescribeVariants is what probes model sizes, so calling it for - // every entry would put a network round trip behind each of the - // ~1200 entries in the gallery. The HasVariants guard keeps an - // ordinary entry exactly as cheap as it was before variants - // existed; only a declaring entry pays, and only on the page it - // appears on. + // Only the cheap declaration flag, never the description itself. + // Describing a variant probes every referenced entry's weight files + // over the network, so doing it here would cost one page load + // (entries x variants) serial round trips; a client that wants the + // description asks /api/models/variants/:id for one entry at a + // time, exactly as it already does for VRAM estimates. + // + // The flag is omitted rather than sent as false so an entry that + // declares nothing stays byte-for-byte what it was before variants + // existed, and so a client never asks about it. if m.HasVariants() { - env := gallery.HostResolveEnv(c.Request().Context(), appConfig.SystemState) - view, describeErr := gallery.DescribeVariants(allModels, m, env) - if describeErr != nil { - // A malformed variant list must not blank the whole gallery - // page; that entry just renders without a picker. - xlog.Warn("could not describe model variants", "model", m.Name, "error", describeErr) - } else if view != nil { - obj["variants"] = view.Variants - obj["auto_variant"] = view.AutoSelected - } + obj["has_variants"] = true } modelsJSON = append(modelsJSON, obj) @@ -825,6 +814,53 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model return c.JSON(200, result) }, adminMiddleware) + // Returns the selectable builds of a single gallery model and which one + // auto-selection would install on this host. Companion to estimate/:id and + // for the same reason: describing variants probes each referenced entry's + // weight files over the network, so the gallery listing must stay free of + // it and the frontend fills pickers in per-entry, on demand. + // + // An entry that declares no variants is not an error; it answers with an + // empty description, so a client that asks anyway gets a well-formed reply + // rather than a 404 it has to special-case. + app.GET("/api/models/variants/:id", func(c echo.Context) error { + modelID, err := url.QueryUnescape(c.Param("id")) + if err != nil { + return c.JSON(http.StatusBadRequest, map[string]any{"error": "invalid model ID"}) + } + + models, err := gallery.AvailableGalleryModelsCached(appConfig.Galleries, appConfig.SystemState) + if err != nil { + return c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error()}) + } + + model := gallery.FindGalleryElement(models, modelID) + if model == nil { + return c.JSON(http.StatusNotFound, map[string]any{"error": "model not found"}) + } + + // An allocated slice rather than the zero value, so "nothing + // selectable" serializes as [] and a client can iterate the reply + // without first distinguishing it from null. + empty := gallery.EntryVariants{Variants: []gallery.VariantView{}} + + // The full, unpaginated list: a variant references another gallery + // entry by name and that entry need not be anywhere near this one. + env := gallery.HostResolveEnv(c.Request().Context(), appConfig.SystemState) + view, err := gallery.DescribeVariants(models, model, env) + if err != nil { + // A malformed variant list must not break the picker; the entry + // just reports nothing selectable and installs as-is. + xlog.Debug("could not describe model variants", "model", modelID, "error", err) + return c.JSON(200, empty) + } + if view == nil { + return c.JSON(200, empty) + } + + return c.JSON(200, view) + }, adminMiddleware) + app.POST("/api/models/install/:id", func(c echo.Context) error { galleryID := c.Param("id") // URL decode the gallery ID (e.g., "localai%40model" -> "localai@model") diff --git a/core/http/routes/ui_api_variants_test.go b/core/http/routes/ui_api_variants_test.go new file mode 100644 index 000000000000..508a4522327a --- /dev/null +++ b/core/http/routes/ui_api_variants_test.go @@ -0,0 +1,211 @@ +package routes_test + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "sync/atomic" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/application" + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/routes" + "github.com/mudler/LocalAI/core/services/galleryop" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// These specs pin the cost contract of the gallery listing. +// +// Variant description probes every referenced entry's weight files over the +// network. Doing that inline in the listing makes one page load cost +// (entries x variants) serial round trips, so the listing must not do it at +// all: it reports only the cheap `has_variants` flag, and a client that wants +// the description asks for one entry at a time. +// +// The probe counter below is what makes that a behavioral assertion rather +// than a structural one. It counts real HTTP hits on the weight files, so it +// goes red if DescribeVariants becomes reachable from the listing path again, +// through any caller. +var _ = Describe("Model gallery variants API", func() { + var ( + app *echo.Echo + modelsDir string + weightServer *httptest.Server + indexServer *httptest.Server + probes *atomic.Int64 + appConfig *config.ApplicationConfig + ) + + BeforeEach(func() { + var err error + modelsDir, err = os.MkdirTemp("", "ui-api-variants-test-*") + Expect(err).NotTo(HaveOccurred()) + + probes = &atomic.Int64{} + // Stands in for the weight files a variant probe would range-fetch. + // Every hit is a probe; a listing that describes variants inline + // cannot avoid them. + weightServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + probes.Add(1) + w.Header().Set("Content-Length", "1048576") + w.WriteHeader(http.StatusOK) + })) + + index := fmt.Sprintf(` +- name: base-entry + description: An entry that declares variants + backend: llama-cpp + files: + - filename: base.gguf + uri: %s/base.gguf + sha256: "" + variants: + - model: big-entry +- name: big-entry + description: The alternative build + backend: llama-cpp + files: + - filename: big.gguf + uri: %s/big.gguf + sha256: "" +- name: plain-entry + description: An entry that declares nothing + backend: whisper +`, weightServer.URL, weightServer.URL) + + indexServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(index)) + })) + + systemState, err := system.GetSystemState(system.WithModelPath(modelsDir)) + Expect(err).NotTo(HaveOccurred()) + + appConfig = &config.ApplicationConfig{ + Galleries: []config.Gallery{{Name: "test", URL: indexServer.URL + "/index.yaml"}}, + SystemState: systemState, + } + + galleryService := galleryop.NewGalleryService(appConfig, nil) + app = echo.New() + routes.RegisterUIAPIRoutes(app, config.NewModelConfigLoader(modelsDir), model.NewModelLoader(systemState), appConfig, + galleryService, galleryop.NewOpCache(galleryService), &application.Application{}, + func(next echo.HandlerFunc) echo.HandlerFunc { return next }) + }) + + AfterEach(func() { + weightServer.Close() + indexServer.Close() + Expect(os.RemoveAll(modelsDir)).To(Succeed()) + }) + + get := func(path string) (int, map[string]any) { + req := httptest.NewRequest(http.MethodGet, path, nil) + rec := httptest.NewRecorder() + app.ServeHTTP(rec, req) + var body map[string]any + Expect(json.Unmarshal(rec.Body.Bytes(), &body)).To(Succeed()) + return rec.Code, body + } + + listing := func() []map[string]any { + code, body := get("/api/models?items=9999") + Expect(code).To(Equal(http.StatusOK)) + raw, ok := body["models"].([]any) + Expect(ok).To(BeTrue(), "listing must return a models array") + out := make([]map[string]any, 0, len(raw)) + for _, m := range raw { + out = append(out, m.(map[string]any)) + } + return out + } + + find := func(models []map[string]any, name string) map[string]any { + for _, m := range models { + if m["name"] == name { + return m + } + } + Fail("no entry named " + name + " in the listing") + return nil + } + + Context("the listing", func() { + It("issues no variant probes at all", func() { + models := listing() + Expect(models).NotTo(BeEmpty()) + + // The whole point. An entry declaring variants must cost the + // listing exactly what an entry declaring none costs it. + Expect(probes.Load()).To(BeZero(), + "the gallery listing probed variant weight files; variant description must not be reachable from the listing path") + }) + + It("omits the described variant payload", func() { + entry := find(listing(), "base-entry") + Expect(entry).NotTo(HaveKey("variants")) + Expect(entry).NotTo(HaveKey("auto_variant")) + }) + + It("reports has_variants so a client knows whether to ask", func() { + models := listing() + Expect(find(models, "base-entry")["has_variants"]).To(BeTrue()) + // An entry declaring nothing must look exactly as it did before + // variants existed, so a client never asks about it. + Expect(find(models, "plain-entry")).NotTo(HaveKey("has_variants")) + }) + }) + + Context("the companion endpoint", func() { + It("returns the description the listing used to carry", func() { + code, body := get("/api/models/variants/test@base-entry") + Expect(code).To(Equal(http.StatusOK)) + + Expect(body).To(HaveKey("auto_selected")) + variants, ok := body["variants"].([]any) + Expect(ok).To(BeTrue()) + Expect(variants).To(HaveLen(2), "the declared variant plus the base") + + byModel := map[string]map[string]any{} + for _, v := range variants { + vm := v.(map[string]any) + byModel[vm["model"].(string)] = vm + } + Expect(byModel).To(HaveKey("big-entry")) + Expect(byModel).To(HaveKey("base-entry")) + Expect(byModel["base-entry"]["is_base"]).To(BeTrue()) + Expect(byModel["big-entry"]["is_base"]).To(BeFalse()) + Expect(byModel["big-entry"]).To(HaveKey("backend")) + Expect(byModel["big-entry"]).To(HaveKey("fits")) + }) + + It("omits memory_bytes entirely when a size cannot be determined", func() { + // The weight server answers without a usable size, so the probe + // comes back unknown. An absent key is the contract: a zero would + // read as 'needs nothing'. + _, body := get("/api/models/variants/test@base-entry") + for _, v := range body["variants"].([]any) { + vm := v.(map[string]any) + if mb, present := vm["memory_bytes"]; present { + Expect(mb).NotTo(BeZero(), "memory_bytes must be omitted rather than serialized as zero") + } + } + }) + + It("returns an empty description for an entry declaring no variants", func() { + code, body := get("/api/models/variants/test@plain-entry") + Expect(code).To(Equal(http.StatusOK)) + Expect(body["variants"]).To(BeEmpty()) + Expect(body["auto_selected"]).To(BeEmpty()) + }) + + It("404s an unknown entry", func() { + code, _ := get("/api/models/variants/test@nope") + Expect(code).To(Equal(http.StatusNotFound)) + }) + }) +}) diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index b02ae336d088..e8b2f26acf97 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -178,18 +178,26 @@ larger ones, or both. Sizes are measured from the model's weights rather than downloaded, and cached. -The gallery listing reports what an entry offers. Entries with variants carry -two extra fields, `variants` and `auto_variant`: +The gallery listing only flags which entries offer variants, with a +`has_variants` field. It deliberately does not describe them: measuring a +variant is a network round trip per referenced build, so describing every +entry inline would make one listing request cost as many round trips as the +whole page has variants. ```bash -curl http://localhost:8080/api/models | jq '.models[] | select(.variants) | - {name, auto_variant, variants}' +curl http://localhost:8080/api/models | jq '.models[] | select(.has_variants) | .name' +``` + +Ask for the description one entry at a time, as the web UI does when you open +a model's variant menu: + +```bash +curl http://localhost:8080/api/models/variants/localai@nanbeige4.1-3b-q4 ``` ```json { - "name": "nanbeige4.1-3b-q4", - "auto_variant": "nanbeige4.1-3b-q8", + "auto_selected": "nanbeige4.1-3b-q8", "variants": [ { "model": "nanbeige4.1-3b-q8", "backend": "llama-cpp", "memory_bytes": 4187593113, "fits": true, "is_base": false }, { "model": "nanbeige4.1-3b-q4", "backend": "llama-cpp", "fits": true, "is_base": true } @@ -197,12 +205,15 @@ curl http://localhost:8080/api/models | jq '.models[] | select(.variants) | } ``` -`auto_variant` is what installing without a choice would pick right now. `fits` +`auto_selected` is what installing without a choice would pick right now. `fits` is whether auto-selection would consider that variant on this machine, and `is_base` marks the entry's own build. `memory_bytes` is omitted entirely, as on the second entry above, when the size could not be measured; read a missing `memory_bytes` as unknown rather than as a free build. +An entry that declares no variants carries no `has_variants` field and answers +this endpoint with an empty list, so a client never has to ask about it. + To install a specific one, pass its name as `variant`: ```bash From 831b127ded9e4c5cd3fbe49760352c182f43ea6b Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 19 Jul 2026 07:52:22 +0000 Subject: [PATCH 24/48] feat(ui): filter the model gallery to entries that declare variants The gallery is heading towards showing parent entries and hiding the individual builds they reference, so a user sees one row per model rather than six quantizations of it. Adoption is a single entry today, so defaulting to that would leave a one-row gallery. This ships the migration-phase inverse instead: the default is untouched, and a toggle narrows the list to only the entries that declare variants. It previews the end state and changes nothing until someone asks for it. The filter is server-side, next to term/tag/backend/capability and above the pagination arithmetic. The listing paginates at 9 items, so narrowing on the client would leave totalPages and availableModels describing the unfiltered set and hand the user empty pages. It selects on HasVariants(), which reads already-loaded metadata, so it issues no variant probes. The parameter is named has_variants after the listing field it selects on, and is compared against "true" like the other boolean query params (all_users, save_checkpoint), so has_variants=false reads as absent. With it omitted the response is byte-for-byte what it was before. The control is the shared Toggle component, matching the fitsFilter toggle already on this page: same wrapper class, same icon and label shape, same localStorage persistence. Unlike fitsFilter it resets to page 1 on change, which a server-side filter has to do. Stacking the toggle with a tag or backend filter easily yields nothing while one entry declares variants, so the empty state now names the variants filter as the cause rather than leaving a user to conclude the gallery is broken. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/http/react-ui/e2e/models-gallery.spec.js | 162 ++++++++++++++++++ .../react-ui/public/locales/de/models.json | 2 + .../react-ui/public/locales/en/models.json | 2 + .../react-ui/public/locales/es/models.json | 2 + .../react-ui/public/locales/id/models.json | 2 + .../react-ui/public/locales/it/models.json | 2 + .../react-ui/public/locales/zh-CN/models.json | 2 + core/http/react-ui/src/pages/Models.jsx | 46 ++++- core/http/routes/ui_api.go | 24 +++ core/http/routes/ui_api_variants_test.go | 82 +++++++++ 10 files changed, 319 insertions(+), 7 deletions(-) diff --git a/core/http/react-ui/e2e/models-gallery.spec.js b/core/http/react-ui/e2e/models-gallery.spec.js index 5d3934710974..8e0d58f00b7c 100644 --- a/core/http/react-ui/e2e/models-gallery.spec.js +++ b/core/http/react-ui/e2e/models-gallery.spec.js @@ -664,3 +664,165 @@ test.describe("Models Gallery - Variant picker", () => { expect(variantUrls).toHaveLength(1); }); }); + +// The gallery is heading towards hiding the individual builds a parent entry +// references. Adoption is one entry, so this toggle is the inverse of that end +// state: off by default and changing nothing, on to preview it. The filter is +// server-side because the listing paginates, so these specs assert on the +// request the page actually sends, not just on the rows it renders. +const VARIANTS_ONLY_RESPONSE = { + ...MOCK_MODELS_RESPONSE, + models: MOCK_MODELS_RESPONSE.models.filter((m) => m.has_variants), + availableModels: 1, + totalPages: 1, + currentPage: 1, +}; + +test.describe("Models Gallery - Has Variants Filter", () => { + let listingUrls; + + const variantsToggle = (page) => + page + .locator("label.filter-bar-group__toggle", { hasText: "Has variants" }) + .locator(".toggle__track"); + + test.beforeEach(async ({ page }) => { + listingUrls = []; + + await page.route("**/api/models*", (route) => { + const url = new URL(route.request().url()); + // Only the listing itself; sibling routes like /api/models/estimate + // must not pollute the record of what the filter sent. + if (url.pathname.endsWith("/api/models")) { + listingUrls.push(url); + } + const onlyVariants = url.searchParams.get("has_variants") === "true"; + const tag = url.searchParams.get("tag"); + let body = MOCK_MODELS_RESPONSE; + if (onlyVariants) { + // Stacking the toggle with a usecase filter is the case that easily + // yields nothing while a single entry declares variants. + body = tag ? EMPTY_FILTERED_RESPONSE : VARIANTS_ONLY_RESPONSE; + } + route.fulfill({ + contentType: "application/json", + body: JSON.stringify(body), + }); + }); + + await page.goto("/app/models"); + await expect(page.locator("th", { hasText: "Backend" })).toBeVisible({ + timeout: 10_000, + }); + }); + + test("the toggle is visible", async ({ page }) => { + await expect(page.getByText("Has variants")).toBeVisible(); + }); + + test("defaults to off, showing everything and sending no filter param", async ({ + page, + }) => { + await expect(page.getByLabel("Has variants")).not.toBeChecked(); + await expect(page.locator("tr", { hasText: "llama-model" })).toBeVisible(); + await expect( + page.locator("tr", { hasText: "whisper-model" }), + ).toBeVisible(); + await expect( + page.locator("tr", { hasText: "stablediffusion-model" }), + ).toBeVisible(); + + // Asserted positively over every listing request, so a param that leaked + // in as has_variants=false would still fail this. + expect(listingUrls.length).toBeGreaterThan(0); + for (const url of listingUrls) { + expect(url.searchParams.has("has_variants")).toBe(false); + } + }); + + test("toggling on sends has_variants=true and narrows the list", async ({ + page, + }) => { + await variantsToggle(page).click(); + + await expect(page.locator("tr", { hasText: "llama-model" })).toBeVisible(); + await expect(page.locator("tr", { hasText: "whisper-model" })).toHaveCount( + 0, + ); + await expect( + page.locator("tr", { hasText: "stablediffusion-model" }), + ).toHaveCount(0); + + const last = listingUrls[listingUrls.length - 1]; + expect(last.searchParams.get("has_variants")).toBe("true"); + }); + + test("toggling off restores the full list and drops the param", async ({ + page, + }) => { + await variantsToggle(page).click(); + await expect(page.locator("tr", { hasText: "whisper-model" })).toHaveCount( + 0, + ); + + await variantsToggle(page).click(); + + await expect( + page.locator("tr", { hasText: "whisper-model" }), + ).toBeVisible(); + await expect( + page.locator("tr", { hasText: "stablediffusion-model" }), + ).toBeVisible(); + + const last = listingUrls[listingUrls.length - 1]; + expect(last.searchParams.has("has_variants")).toBe(false); + }); + + test("resets to page 1 when toggled", async ({ page }) => { + await variantsToggle(page).click(); + const last = listingUrls[listingUrls.length - 1]; + // A filter that narrowed the rows while holding page 3 would strand the + // user on a page the filtered set no longer has. + expect(last.searchParams.get("page") || "1").toBe("1"); + }); + + test("state persists after reload, like the other filters", async ({ + page, + }) => { + await variantsToggle(page).click(); + await page.reload(); + await expect(page.getByLabel("Has variants")).toBeChecked(); + await expect(page.locator("tr", { hasText: "whisper-model" })).toHaveCount( + 0, + ); + }); + + test("names the variants filter in the empty state when it is the cause", async ({ + page, + }) => { + await variantsToggle(page).click(); + await page.locator(".filter-btn", { hasText: "Chat" }).click(); + + await expect(page.locator(".empty-state-title")).toHaveText( + "No models found", + ); + // The generic copy would leave a user thinking the gallery is broken + // rather than that this toggle is doing exactly what it says. + await expect(page.locator(".empty-state-text")).toHaveText( + "No entries declare variants yet. Turn off the variants filter to see the full gallery.", + ); + }); + + test("clear filters turns the toggle back off", async ({ page }) => { + await variantsToggle(page).click(); + await page.locator(".filter-btn", { hasText: "Chat" }).click(); + await expect(page.locator(".empty-state")).toBeVisible(); + + await page.getByRole("button", { name: "Clear filters" }).click(); + + await expect(page.getByLabel("Has variants")).not.toBeChecked(); + await expect( + page.locator("tr", { hasText: "whisper-model" }), + ).toBeVisible(); + }); +}); diff --git a/core/http/react-ui/public/locales/de/models.json b/core/http/react-ui/public/locales/de/models.json index 9142a1b8b624..b1f73320b1a5 100644 --- a/core/http/react-ui/public/locales/de/models.json +++ b/core/http/react-ui/public/locales/de/models.json @@ -24,6 +24,7 @@ "embedding": "Embedding", "rerank": "Rerank", "fitsGpu": "Passt in die GPU", + "hasVariants": "Mit Varianten", "allBackends": "Alle Backends", "searchBackends": "Backends suchen..." }, @@ -71,6 +72,7 @@ "empty": { "title": "Keine Modelle gefunden", "withFilters": "Keine Modelle entsprechen den aktuellen Such- oder Filterkriterien.", + "withVariantsFilter": "Noch keine Eintraege deklarieren Varianten. Schalte den Varianten-Filter aus, um die vollstaendige Galerie zu sehen.", "noFilters": "Die Modellgalerie ist leer." }, "deleteDialog": { diff --git a/core/http/react-ui/public/locales/en/models.json b/core/http/react-ui/public/locales/en/models.json index 9c750774f119..5bcb7db14785 100644 --- a/core/http/react-ui/public/locales/en/models.json +++ b/core/http/react-ui/public/locales/en/models.json @@ -43,6 +43,7 @@ "vad": "VAD", "ner": "NER", "fitsGpu": "Fits in GPU", + "hasVariants": "Has variants", "allBackends": "All Backends", "searchBackends": "Search backends..." }, @@ -90,6 +91,7 @@ "empty": { "title": "No models found", "withFilters": "No models match your current search or filters.", + "withVariantsFilter": "No entries declare variants yet. Turn off the variants filter to see the full gallery.", "noFilters": "The model gallery is empty." }, "deleteDialog": { diff --git a/core/http/react-ui/public/locales/es/models.json b/core/http/react-ui/public/locales/es/models.json index 69ca19daec99..e0005b6f1552 100644 --- a/core/http/react-ui/public/locales/es/models.json +++ b/core/http/react-ui/public/locales/es/models.json @@ -24,6 +24,7 @@ "embedding": "Embedding", "rerank": "Rerank", "fitsGpu": "Cabe en la GPU", + "hasVariants": "Con variantes", "allBackends": "Todos los backends", "searchBackends": "Buscar backends..." }, @@ -71,6 +72,7 @@ "empty": { "title": "No se encontraron modelos", "withFilters": "Ningún modelo coincide con la búsqueda o filtros actuales.", + "withVariantsFilter": "Ninguna entrada declara variantes todavia. Desactiva el filtro de variantes para ver la galeria completa.", "noFilters": "La galería de modelos está vacía." }, "deleteDialog": { diff --git a/core/http/react-ui/public/locales/id/models.json b/core/http/react-ui/public/locales/id/models.json index 3acff3afb249..4c0b582fad09 100644 --- a/core/http/react-ui/public/locales/id/models.json +++ b/core/http/react-ui/public/locales/id/models.json @@ -31,6 +31,7 @@ "detection": "Deteksi", "vad": "VAD", "fitsGpu": "Muat di GPU", + "hasVariants": "Punya varian", "allBackends": "Semua Backend", "searchBackends": "Cari backends..." }, @@ -78,6 +79,7 @@ "empty": { "title": "Model tidak ditemukan", "withFilters": "Tidak ada model yang cocok dengan pencarian atau filter Anda.", + "withVariantsFilter": "Belum ada entri yang mendeklarasikan varian. Matikan filter varian untuk melihat seluruh galeri.", "noFilters": "Galeri model kosong." }, "deleteDialog": { diff --git a/core/http/react-ui/public/locales/it/models.json b/core/http/react-ui/public/locales/it/models.json index 15eca936fdc9..dbfea4f82d7e 100644 --- a/core/http/react-ui/public/locales/it/models.json +++ b/core/http/react-ui/public/locales/it/models.json @@ -24,6 +24,7 @@ "embedding": "Embedding", "rerank": "Rerank", "fitsGpu": "Entra nella GPU", + "hasVariants": "Con varianti", "allBackends": "Tutti i backend", "searchBackends": "Cerca backend..." }, @@ -71,6 +72,7 @@ "empty": { "title": "Nessun modello trovato", "withFilters": "Nessun modello corrisponde ai filtri attuali.", + "withVariantsFilter": "Nessuna voce dichiara varianti al momento. Disattiva il filtro varianti per vedere l'intera galleria.", "noFilters": "La galleria modelli è vuota." }, "deleteDialog": { diff --git a/core/http/react-ui/public/locales/zh-CN/models.json b/core/http/react-ui/public/locales/zh-CN/models.json index 88cd49d7d998..a3d064e5e4ed 100644 --- a/core/http/react-ui/public/locales/zh-CN/models.json +++ b/core/http/react-ui/public/locales/zh-CN/models.json @@ -24,6 +24,7 @@ "embedding": "嵌入", "rerank": "重排", "fitsGpu": "适合 GPU", + "hasVariants": "含变体", "allBackends": "所有后端", "searchBackends": "搜索后端..." }, @@ -71,6 +72,7 @@ "empty": { "title": "未找到模型", "withFilters": "没有模型与当前搜索或筛选条件匹配。", + "withVariantsFilter": "目前没有条目声明变体。关闭变体筛选即可查看完整模型库。", "noFilters": "模型库为空。" }, "deleteDialog": { diff --git a/core/http/react-ui/src/pages/Models.jsx b/core/http/react-ui/src/pages/Models.jsx index 75375632b15e..7e211622dfbc 100644 --- a/core/http/react-ui/src/pages/Models.jsx +++ b/core/http/react-ui/src/pages/Models.jsx @@ -22,6 +22,7 @@ import React from 'react' const CONTEXT_SIZES = [8192, 16384, 32768, 65536, 131072, 262144] const CONTEXT_LABELS = ['8K', '16K', '32K', '64K', '128K', '256K'] const FITS_FILTER_STORAGE_KEY = 'localai-models-fits-filter' +const VARIANTS_FILTER_STORAGE_KEY = 'localai-models-has-variants-filter' const FILTERS = [ @@ -86,6 +87,16 @@ export default function Models() { return false } }) + // Narrows the listing to entries that declare variants. Server-side, unlike + // fitsFilter, because the listing paginates and a client-side narrowing + // would leave the page count describing the unfiltered set. + const [variantsFilter, setVariantsFilter] = useState(() => { + try { + return localStorage.getItem(VARIANTS_FILTER_STORAGE_KEY) === '1' + } catch { + return false + } + }) // Total GPU memory for "fits" check const totalGpuMemory = resources?.aggregate?.total_memory || 0 @@ -97,6 +108,7 @@ export default function Models() { const filtersVal = params.filters !== undefined ? params.filters : filters const sortVal = params.sort !== undefined ? params.sort : sort const backendVal = params.backendFilter !== undefined ? params.backendFilter : backendFilter + const variantsVal = params.variantsFilter !== undefined ? params.variantsFilter : variantsFilter const queryParams = { page: params.page || page, items: 9, @@ -104,6 +116,9 @@ export default function Models() { if (filtersVal.length > 0) queryParams.tag = filtersVal.join(',') if (searchVal) queryParams.term = searchVal if (backendVal) queryParams.backend = backendVal + // Omitted entirely when off, so the default request is byte-for-byte + // what it was before the toggle existed. + if (variantsVal) queryParams.has_variants = 'true' if (sortVal) { queryParams.sort = sortVal queryParams.order = params.order || order @@ -121,11 +136,11 @@ export default function Models() { } finally { setLoading(false) } - }, [page, search, filters, sort, order, backendFilter, addToast, t]) + }, [page, search, filters, sort, order, backendFilter, variantsFilter, addToast, t]) useEffect(() => { fetchModels() - }, [page, filters, sort, order, backendFilter]) + }, [page, filters, sort, order, backendFilter, variantsFilter]) // Fetch backend→usecase mapping once on mount useEffect(() => { @@ -290,6 +305,14 @@ export default function Models() { } }, [fitsFilter]) + useEffect(() => { + try { + localStorage.setItem(VARIANTS_FILTER_STORAGE_KEY, variantsFilter ? '1' : '0') + } catch { + // Ignore storage errors (e.g., private browsing restrictions). + } + }, [variantsFilter]) + const visibleModels = models.filter((model) => { if (!fitsFilter) return true const name = model.name || model.id @@ -361,8 +384,16 @@ export default function Models() { ) })} + {totalGpuMemory > 0 && ( -
@@ -408,12 +438,14 @@ export default function Models() {

{t('empty.title')}

- {search || filters.length > 0 || backendFilter || fitsFilter ? t('empty.withFilters') : t('empty.noFilters')} + {variantsFilter + ? t('empty.withVariantsFilter') + : search || filters.length > 0 || backendFilter || fitsFilter ? t('empty.withFilters') : t('empty.noFilters')}

- {(search || filters.length > 0 || backendFilter || fitsFilter) && ( + {(search || filters.length > 0 || backendFilter || fitsFilter || variantsFilter) && ( diff --git a/core/http/routes/ui_api.go b/core/http/routes/ui_api.go index ec5f4258ea89..b3dcc8ce1284 100644 --- a/core/http/routes/ui_api.go +++ b/core/http/routes/ui_api.go @@ -480,6 +480,30 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model models = filtered } + // Narrow to entries that declare variants, i.e. the ones the listing + // marks with has_variants. + // + // This is the inverse of where the gallery is heading. The end state + // hides the individual builds a parent entry references, so a user sees + // one row per model rather than one per quantization. Adoption is still + // a single entry, so defaulting to that today would empty the gallery. + // Opt-in instead: with the parameter absent the response is exactly + // what it was, and the toggle is a preview of the end state that costs + // nothing until someone asks for it. + // + // Server-side because the listing paginates at 9 items; filtering the + // current page in the client would leave the page count describing the + // unfiltered set and hand the user empty pages. + if c.QueryParam("has_variants") == "true" { + filtered := make(gallery.GalleryElements[*gallery.GalleryModel], 0, len(models)) + for _, m := range models { + if m.HasVariants() { + filtered = append(filtered, m) + } + } + models = filtered + } + // Capability filters are derived from the effective gallery model // configuration. In particular, voice cloning is variant-sensitive, so // filtering by the TTS usecase or backend name alone would advertise diff --git a/core/http/routes/ui_api_variants_test.go b/core/http/routes/ui_api_variants_test.go index 508a4522327a..994caee07895 100644 --- a/core/http/routes/ui_api_variants_test.go +++ b/core/http/routes/ui_api_variants_test.go @@ -208,4 +208,86 @@ var _ = Describe("Model gallery variants API", func() { Expect(code).To(Equal(http.StatusNotFound)) }) }) + + // The gallery is heading towards hiding the individual builds a parent + // entry references. Adoption is one entry, so the toggle is the inverse of + // that end state: off by default and changing nothing, on to preview it. + Context("the has_variants listing filter", func() { + names := func(path string) []string { + code, body := get(path) + Expect(code).To(Equal(http.StatusOK)) + raw, ok := body["models"].([]any) + Expect(ok).To(BeTrue(), "listing must return a models array") + out := make([]string, 0, len(raw)) + for _, m := range raw { + out = append(out, m.(map[string]any)["name"].(string)) + } + return out + } + + rawBody := func(path string) []byte { + req := httptest.NewRequest(http.MethodGet, path, nil) + rec := httptest.NewRecorder() + app.ServeHTTP(rec, req) + Expect(rec.Code).To(Equal(http.StatusOK)) + return rec.Body.Bytes() + } + + It("returns every entry when off", func() { + Expect(names("/api/models?items=9999")).To(ConsistOf("base-entry", "big-entry", "plain-entry")) + }) + + It("returns only entries that declare variants when on", func() { + // big-entry is the build base-entry references, so it must drop + // out even though it is a perfectly valid entry on its own. + Expect(names("/api/models?items=9999&has_variants=true")).To(ConsistOf("base-entry")) + }) + + It("leaves the response untouched for any value other than true", func() { + // Default off has to mean off, so an explicit false and an + // unparseable value must both behave as absent rather than as a + // truthy presence check. + base := rawBody("/api/models?items=9999") + Expect(rawBody("/api/models?items=9999&has_variants=false")).To(Equal(base)) + Expect(rawBody("/api/models?items=9999&has_variants=1")).To(Equal(base)) + }) + + It("serializes a non-declaring entry exactly as it did before", func() { + // The whole promise of the migration phase: a user who never + // touches the toggle sees byte-for-byte what they saw before. + entry := find(listing(), "plain-entry") + Expect(entry).NotTo(HaveKey("has_variants")) + Expect(entry).NotTo(HaveKey("variants")) + Expect(entry).NotTo(HaveKey("auto_variant")) + }) + + It("composes with the backend filter rather than replacing it", func() { + // base-entry declares variants and is llama-cpp; plain-entry is + // whisper and declares none. If either filter overwrote the other, + // the whisper case would return base-entry or plain-entry instead + // of nothing. + Expect(names("/api/models?items=9999&has_variants=true&backend=llama-cpp")).To(ConsistOf("base-entry")) + Expect(names("/api/models?items=9999&has_variants=true&backend=whisper")).To(BeEmpty()) + Expect(names("/api/models?items=9999&backend=whisper")).To(ConsistOf("plain-entry")) + }) + + It("composes with the search term", func() { + Expect(names("/api/models?items=9999&has_variants=true&term=base")).To(ConsistOf("base-entry")) + Expect(names("/api/models?items=9999&has_variants=true&term=plain")).To(BeEmpty()) + }) + + It("reports the filtered total so pagination stays honest", func() { + // The listing paginates at 9, so a filter that narrowed the rows + // without narrowing the count would hand the user empty pages. + _, body := get("/api/models?items=9999&has_variants=true") + Expect(body["availableModels"]).To(BeEquivalentTo(1)) + Expect(body["totalPages"]).To(BeEquivalentTo(1)) + }) + + It("still issues no variant probes when filtering", func() { + names("/api/models?items=9999&has_variants=true") + Expect(probes.Load()).To(BeZero(), + "the filter must select on declared metadata, not by describing variants") + }) + }) }) From b35d630cf6eb9b5d08c5729179d9d305ab2ff508 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 19 Jul 2026 11:05:05 +0000 Subject: [PATCH 25/48] fix(ui): render gallery model descriptions as Markdown Gallery descriptions are Markdown, but the React UI dumped them raw, so a model whose description opens with an ATX heading showed a literal "# Qwen3.6-27B [](https://chat.qwen.ai)" in the list. Full-description areas now render through renderMarkdown (marked + DOMPurify), matching how Backends.jsx and the Manage detail panels already handle the same content: - Models.jsx expanded detail row - VoiceLibrary.jsx voice detail header The truncated one-line previews must not render block Markdown: a leading "#" would become an

and wreck the row height and rhythm. They get a new stripMarkdown() helper instead, which reduces Markdown to a single line of readable plain text. It is used for the cell text and for the title tooltip, since a tooltip full of "[](url)" is no better than a cell full of it: - Models.jsx gallery table description cell - Manage.jsx model and backend resource-row descriptions stripMarkdown walks marked's lexer output rather than running regexes over the source, so what it strips is by construction what renderMarkdown would have rendered, and it needs no new dependency. Output lands in JSX text nodes, so React escapes it; no new dangerouslySetInnerHTML beyond the two full-description sites, both of which run DOMPurify. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/http/react-ui/e2e/models-gallery.spec.js | 93 +++++++++++++++++++ core/http/react-ui/src/pages/Manage.jsx | 19 ++-- core/http/react-ui/src/pages/Models.jsx | 25 +++-- core/http/react-ui/src/pages/VoiceLibrary.jsx | 5 +- core/http/react-ui/src/utils/markdown.js | 47 ++++++++++ 5 files changed, 174 insertions(+), 15 deletions(-) diff --git a/core/http/react-ui/e2e/models-gallery.spec.js b/core/http/react-ui/e2e/models-gallery.spec.js index 8e0d58f00b7c..389e8d0dda45 100644 --- a/core/http/react-ui/e2e/models-gallery.spec.js +++ b/core/http/react-ui/e2e/models-gallery.spec.js @@ -826,3 +826,96 @@ test.describe("Models Gallery - Has Variants Filter", () => { ).toBeVisible(); }); }); + +// Gallery descriptions are third-party Markdown. They used to be dumped raw +// into the UI, so a model whose description opened with an ATX heading showed +// a literal "# Name [](url)" in the list. +const MARKDOWN_DESCRIPTION = + "# Qwen3.6-27B\n\nChat with it at [the Qwen site](https://chat.qwen.ai) for **free**."; +const MARKDOWN_MODELS_RESPONSE = { + ...MOCK_MODELS_RESPONSE, + models: [ + { + name: "markdown-model", + description: MARKDOWN_DESCRIPTION, + backend: "llama-cpp", + installed: false, + tags: ["chat"], + }, + { + name: "no-description-model", + description: "", + backend: "llama-cpp", + installed: false, + tags: ["chat"], + }, + ], + availableModels: 2, + installedModels: 0, +}; + +test.describe("Models Gallery - Markdown descriptions", () => { + test.beforeEach(async ({ page }) => { + await page.route("**/api/models*", (route) => { + route.fulfill({ + contentType: "application/json", + body: JSON.stringify(MARKDOWN_MODELS_RESPONSE), + }); + }); + await page.goto("/app/models"); + await expect(page.locator("th", { hasText: "Backend" })).toBeVisible({ + timeout: 10_000, + }); + }); + + test("table cell shows the description as clean text, not raw Markdown", async ({ + page, + }) => { + const row = page.locator("tr", { hasText: "markdown-model" }); + const cell = row.locator("div[title]", { hasText: "Qwen3.6-27B" }); + + await expect(cell).toHaveText( + "Qwen3.6-27B Chat with it at the Qwen site for free.", + ); + // The syntax itself must be gone, not merely rendered somewhere. + await expect(cell).not.toContainText("#"); + await expect(cell).not.toContainText("[]("); + await expect(cell).not.toContainText("**"); + await expect(cell).not.toContainText("https://chat.qwen.ai"); + // A block element here would blow up the row height. + await expect(cell.locator("h1")).toHaveCount(0); + }); + + test("title tooltip carries the stripped text, not raw Markdown", async ({ + page, + }) => { + const row = page.locator("tr", { hasText: "markdown-model" }); + const cell = row.locator("div[title]", { hasText: "Qwen3.6-27B" }); + + await expect(cell).toHaveAttribute( + "title", + "Qwen3.6-27B Chat with it at the Qwen site for free.", + ); + }); + + test("expanded detail row renders the description as real markup", async ({ + page, + }) => { + await page.locator("tr", { hasText: "markdown-model" }).click(); + + const detail = page.locator('td[colspan="8"]'); + await expect(detail.locator("h1", { hasText: "Qwen3.6-27B" })).toBeVisible(); + const link = detail.locator('a[href="https://chat.qwen.ai"]'); + await expect(link).toBeVisible(); + await expect(link).toHaveText("the Qwen site"); + await expect(detail.locator("strong", { hasText: "free" })).toBeVisible(); + }); + + test("a model without a description still shows the placeholder", async ({ + page, + }) => { + const row = page.locator("tr", { hasText: "no-description-model" }); + await expect(row).toBeVisible(); + await expect(row.locator("div[title='']")).toHaveText("—"); + }); +}); diff --git a/core/http/react-ui/src/pages/Manage.jsx b/core/http/react-ui/src/pages/Manage.jsx index a5e19dbe8298..9f612ea779a9 100644 --- a/core/http/react-ui/src/pages/Manage.jsx +++ b/core/http/react-ui/src/pages/Manage.jsx @@ -17,7 +17,7 @@ import { useModels } from '../hooks/useModels' import { useGalleryEnrichment } from '../hooks/useGalleryEnrichment' import { useOperations } from '../hooks/useOperations' import { backendControlApi, modelsApi, backendsApi, systemApi, nodesApi } from '../utils/api' -import { renderMarkdown } from '../utils/markdown' +import { renderMarkdown, stripMarkdown } from '../utils/markdown' import { safeHref } from '../utils/url' import { CAP_CHAT, CAP_COMPLETION, CAP_IMAGE, CAP_VIDEO, CAP_TTS, @@ -121,6 +121,15 @@ function formatBackendVersion(metadata) { return { label: '—', full: '' } } +// Gallery descriptions are Markdown. The row preview is a single truncated +// line, so it shows the text without the syntax; the full Markdown is rendered +// in the expanded detail panel instead. +function ResourceRowDesc({ description }) { + const text = stripMarkdown(description) + if (!text) return null + return {text} +} + export default function Manage() { const { addToast } = useOutletContext() const navigate = useNavigate() @@ -640,9 +649,7 @@ export default function Manage() {
{model.id} {enriched?.description && ( - - {enriched.description} - + )}
@@ -957,9 +964,7 @@ export default function Manage() { )}

{(enriched?.description) && ( - - {enriched.description} - + )}
diff --git a/core/http/react-ui/src/pages/Models.jsx b/core/http/react-ui/src/pages/Models.jsx index 7e211622dfbc..1f0830bbce3a 100644 --- a/core/http/react-ui/src/pages/Models.jsx +++ b/core/http/react-ui/src/pages/Models.jsx @@ -16,6 +16,7 @@ import ResponsiveTable from '../components/ResponsiveTable' import RecommendedModels from '../components/RecommendedModels' import Popover from '../components/Popover' import { formatBytes } from '../utils/format' +import { renderMarkdown, stripMarkdown } from '../utils/markdown' import React from 'react' @@ -529,12 +530,19 @@ export default function Models() { {/* Description */} -
- {model.description || '—'} -
+ {(() => { + // Gallery descriptions are Markdown. This cell is a single + // truncated line, so it gets the text without the syntax. + const desc = stripMarkdown(model.description) + return ( +
+ {desc || '—'} +
+ ) + })()} {/* Backend */} @@ -796,7 +804,10 @@ function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setE {model.description && ( - {model.description} +
)} diff --git a/core/http/react-ui/src/pages/VoiceLibrary.jsx b/core/http/react-ui/src/pages/VoiceLibrary.jsx index 642042b522bd..921838aa8ca2 100644 --- a/core/http/react-ui/src/pages/VoiceLibrary.jsx +++ b/core/http/react-ui/src/pages/VoiceLibrary.jsx @@ -14,6 +14,7 @@ import { useVoiceCloningGallery, useVoiceProfiles } from '../hooks/useVoiceProfi import { CAP_TTS } from '../utils/capabilities' import { modelsApi, voiceProfilesApi } from '../utils/api' import { copyToClipboard } from '../utils/clipboard' +import { renderMarkdown } from '../utils/markdown' const ROW_BARS = [35, 58, 78, 44, 68, 92, 55, 72, 38, 82, 64, 46, 74, 52, 88, 42, 66, 48] @@ -317,7 +318,9 @@ JSON` : ''
{t('voiceLibrary.detail.eyebrow')}

{selected.name}

- {selected.description &&

{selected.description}

} + {selected.description && ( +
+ )}
diff --git a/core/http/react-ui/src/utils/markdown.js b/core/http/react-ui/src/utils/markdown.js index 24f253e2b964..79f6d30139ce 100644 --- a/core/http/react-ui/src/utils/markdown.js +++ b/core/http/react-ui/src/utils/markdown.js @@ -20,6 +20,53 @@ export function renderMarkdown(text) { return DOMPurify.sanitize(html) } +// Recursively pull the human-readable text out of a marked token, dropping the +// syntax that carries it. Link/image tokens keep their label and lose the URL; +// code keeps its literal content; raw HTML is dropped rather than leaked as +// angle-bracket noise into a table cell. +function tokenToPlainText(token) { + if (!token) return '' + switch (token.type) { + case 'space': + case 'hr': + case 'br': + case 'html': + return ' ' + case 'code': + case 'codespan': + case 'image': + return token.text || '' + case 'list': + return (token.items || []).map(tokenToPlainText).join(' ') + case 'table': + return [...(token.header || []), ...(token.rows || []).flat()] + .map(tokenToPlainText) + .join(' ') + default: + break + } + if (Array.isArray(token.tokens) && token.tokens.length > 0) { + return token.tokens.map(tokenToPlainText).join('') + } + return token.text || '' +} + +// Reduce Markdown to a single line of readable plain text. Used by the +// truncated one-line description cells and their `title` tooltips, where +// rendering real Markdown would turn a leading `#` into an

and wreck the +// row rhythm. Output lands in JSX text nodes, so React escapes it. +export function stripMarkdown(text) { + if (!text) return '' + let plain + try { + plain = marked.lexer(text).map(tokenToPlainText).join(' ') + } catch { + // A malformed gallery description must never break the row it labels. + plain = text + } + return plain.replace(/\s+/g, ' ').trim() +} + export function highlightAll(element) { if (!element) return element.querySelectorAll('pre code').forEach((block) => { From 08a434da4f9a3b41e9e228354e23874c00d39902 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 19 Jul 2026 12:34:04 +0000 Subject: [PATCH 26/48] fix(ui): strip Markdown from the backends table description cell Commit b35d630cf fixed this for gallery models but left the Backends admin page with the same asymmetry: its detail panel renders the description through renderMarkdown, while the collapsed table row dumped the raw gallery string into both the cell body and the title tooltip. That is user-visible. 40 of the 949 entries in backend/index.yaml carry Markdown - insightface uses inline code backticks, others use lists and links - and backend descriptions also contain embedded newlines, so the one-line cell showed literal syntax. The cell now runs stripMarkdown over the description once and uses the result for the text and the title, matching Models.jsx and the ResourceRowDesc component in Manage.jsx. The '-' placeholder is preserved, and now also fires when a description reduces to nothing after stripping. The detail panel is untouched and no new dangerouslySetInnerHTML is introduced: stripMarkdown output lands in a JSX text node, so React escapes it. Three Playwright specs cover it: a description with a heading, inline code and a link renders as clean text with no literal syntax and no block element in the cell, the title tooltip carries the same stripped text, and a backend without a description still shows the placeholder. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- .../react-ui/e2e/backends-management.spec.js | 51 +++++++++++++++++++ core/http/react-ui/src/pages/Backends.jsx | 24 ++++++--- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/core/http/react-ui/e2e/backends-management.spec.js b/core/http/react-ui/e2e/backends-management.spec.js index c34d505338fe..11b1791891ac 100644 --- a/core/http/react-ui/e2e/backends-management.spec.js +++ b/core/http/react-ui/e2e/backends-management.spec.js @@ -26,3 +26,54 @@ test.describe('Backends management page', () => { await expect(page.getByPlaceholder('oci://quay.io/example/backend:latest')).toBeVisible() }) }) + +// Backend gallery descriptions are Markdown too: 40 of the entries in +// backend/index.yaml carry headings, inline code, lists or links, and they used +// to be dumped raw into the truncated table cell. +const MARKDOWN_DESCRIPTION = + '# InsightFace\n\nUse `insightface` for face analysis. See [the docs](https://example.com/docs) for **details**.' +const STRIPPED_DESCRIPTION = + 'InsightFace Use insightface for face analysis. See the docs for details.' + +test.describe('Backends management page - Markdown descriptions', () => { + test.beforeEach(async ({ page }) => { + await page.route('**/api/backends*', (route) => { + route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + backends: [ + { name: 'markdown-backend', description: MARKDOWN_DESCRIPTION, installed: false }, + { name: 'plain-backend', description: '', installed: false }, + ], + }), + }) + }) + await page.goto('/app/backends') + await expect(page.locator('th', { hasText: 'Description' })).toBeVisible({ timeout: 10_000 }) + }) + + test('table cell shows the description as clean text, not raw Markdown', async ({ page }) => { + const cell = page.locator('tr', { hasText: 'markdown-backend' }).locator('span[title]', { hasText: 'InsightFace' }) + + await expect(cell).toHaveText(STRIPPED_DESCRIPTION) + // The syntax itself must be gone, not merely rendered somewhere. + await expect(cell).not.toContainText('#') + await expect(cell).not.toContainText('`') + await expect(cell).not.toContainText('**') + await expect(cell).not.toContainText('https://example.com/docs') + // A block element here would blow up the row height. + await expect(cell.locator('h1')).toHaveCount(0) + }) + + test('title tooltip carries the stripped text, not raw Markdown', async ({ page }) => { + const cell = page.locator('tr', { hasText: 'markdown-backend' }).locator('span[title]', { hasText: 'InsightFace' }) + + await expect(cell).toHaveAttribute('title', STRIPPED_DESCRIPTION) + }) + + test('a backend with no description still shows the placeholder', async ({ page }) => { + const row = page.locator('tr', { hasText: 'plain-backend' }) + + await expect(row.locator('span[title=""]')).toHaveText('-') + }) +}) diff --git a/core/http/react-ui/src/pages/Backends.jsx b/core/http/react-ui/src/pages/Backends.jsx index b85ba6fe2ca3..deb9385500c2 100644 --- a/core/http/react-ui/src/pages/Backends.jsx +++ b/core/http/react-ui/src/pages/Backends.jsx @@ -8,7 +8,7 @@ import { useOperations } from '../hooks/useOperations' import { useDistributedMode } from '../hooks/useDistributedMode' import LoadingSpinner from '../components/LoadingSpinner' import PageHeader from '../components/PageHeader' -import { renderMarkdown } from '../utils/markdown' +import { renderMarkdown, stripMarkdown } from '../utils/markdown' import { safeHref } from '../utils/url' import ConfirmDialog from '../components/ConfirmDialog' import Toggle from '../components/Toggle' @@ -550,13 +550,21 @@ export default function Backends() { {/* Description */} - - {b.description || '-'} - + {(() => { + // Gallery descriptions are Markdown. This cell is a single + // truncated line, so it gets the text without the syntax; + // the full Markdown is rendered in the detail panel instead. + const desc = stripMarkdown(b.description) + return ( + + {desc || '-'} + + ) + })()} {/* Repository */} From 8130d7ce9d405d371f1a612d2746014dbb3142e2 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 19 Jul 2026 13:07:25 +0000 Subject: [PATCH 27/48] ui(models): polish the variant detail view and scope rendered Markdown The gallery detail pane rendered every field through the same two-column label/value row, including the description. Multi-paragraph prose in a value cell ran eight rows tall at the top of the pane on a ~1200px measure, breaking the grid's rhythm exactly where the eye enters. Move it into its own full-width block above the table, capped at a 68ch measure, keeping the label. Rendered Markdown had no scoped typography anywhere in the app, so a description opening with `#` inherited the browser default 2em inside a 13px surface while a `##` further down was indistinguishable from body text. Add a reusable .markdown-body block mapping h1-h6, paragraphs, lists, links, code, blockquotes, images and tables onto the existing type scale, and apply it to every renderMarkdown() consumer: the models detail, the backends detail, both Manage details and the voice library detail. Rebalance the variants list so the name leads. Backend and size drop from badge/secondary weight to muted metadata; the FITS badge goes entirely, since it was true of nearly every row and so said nothing, while the variant that does not fit keeps a warning badge and a dimmed name. AUTO-SELECTED stays marked because it answers what a plain Install produces. Rows share the parent's grid tracks via subgrid so name, backend, size and status line up down the list instead of raggedly following name length. Finally, make each variant row actionable. It looked like a list of choices but was inert text, with per-variant install hidden behind the split-button chevron elsewhere; each row is now a button onto the existing handleInstall(modelId, variant) path, with hover, keyboard focus and a disabled state while an install is in flight. Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Ettore Di Giacinto --- core/http/react-ui/e2e/models-gallery.spec.js | 129 +++++++++- .../react-ui/public/locales/en/models.json | 4 +- core/http/react-ui/src/App.css | 230 ++++++++++++++++++ core/http/react-ui/src/pages/Backends.jsx | 2 +- core/http/react-ui/src/pages/Manage.jsx | 4 +- core/http/react-ui/src/pages/Models.jsx | 72 ++++-- core/http/react-ui/src/pages/VoiceLibrary.jsx | 2 +- 7 files changed, 410 insertions(+), 33 deletions(-) diff --git a/core/http/react-ui/e2e/models-gallery.spec.js b/core/http/react-ui/e2e/models-gallery.spec.js index 389e8d0dda45..7f3aec4d710a 100644 --- a/core/http/react-ui/e2e/models-gallery.spec.js +++ b/core/http/react-ui/e2e/models-gallery.spec.js @@ -663,6 +663,64 @@ test.describe("Models Gallery - Variant picker", () => { // Expanding is the second trigger point, so it pays for exactly one fetch. expect(variantUrls).toHaveLength(1); }); + + test("the variant rows line up as columns", async ({ page }) => { + await variantRow(page).click(); + const rows = page.locator(".variant-row"); + await expect(rows).toHaveCount(4); + const columns = await rows.evaluateAll((els) => + els.map((el) => ({ + backend: el.querySelector(".variant-row__backend").getBoundingClientRect().x, + size: el.querySelector(".variant-row__size").getBoundingClientRect().right, + })), + ); + // Names differ in length, so without shared tracks each row would start + // its backend at a different x. Sub-pixel rounding is the only tolerance. + for (const c of columns) { + expect(Math.abs(c.backend - columns[0].backend)).toBeLessThan(1.5); + expect(Math.abs(c.size - columns[0].size)).toBeLessThan(1.5); + } + }); + + test("only the informative status is badged", async ({ page }) => { + await variantRow(page).click(); + const detail = page.locator('td[colspan="8"]'); + await expect(detail.locator(".variant-row")).toHaveCount(4); + // "Fits" was true of three rows out of four and said nothing; the row that + // does not fit is the one worth marking. + await expect(detail.getByText("Fits", { exact: true })).toHaveCount(0); + const unfit = detail.locator(".variant-row--unfit"); + await expect(unfit).toHaveCount(1); + await expect(unfit).toContainText("llama-model-f16"); + await expect(unfit.locator(".badge-warning")).toHaveText("Does not fit"); + // Auto-selected still answers "what do I get if I just hit Install". + await expect( + detail.locator(".variant-row", { hasText: "llama-model-q8" }), + ).toContainText("Auto-selected"); + }); + + test("clicking a variant row installs that variant", async ({ page }) => { + await variantRow(page).click(); + await page + .locator(".variant-row", { hasText: "llama-model-mlx" }) + .click(); + await expect.poll(() => installUrls.length).toBe(1); + expect(installUrls[0]).toContain("variant=llama-model-mlx"); + }); + + test("a variant row is reachable and actionable from the keyboard", async ({ + page, + }) => { + await variantRow(page).click(); + const row = page.locator(".variant-row", { hasText: "llama-model-f16" }); + await row.focus(); + // A build that does not fit stays installable: the explicit choice is an + // override the server honours. + await expect(row).toBeFocused(); + await page.keyboard.press("Enter"); + await expect.poll(() => installUrls.length).toBe(1); + expect(installUrls[0]).toContain("variant=llama-model-f16"); + }); }); // The gallery is heading towards hiding the individual builds a parent entry @@ -842,6 +900,14 @@ const MARKDOWN_MODELS_RESPONSE = { installed: false, tags: ["chat"], }, + { + name: "headings-model", + description: + "# Top Heading\n\nBody copy.\n\n## Sub Heading\n\nMore body copy.", + backend: "llama-cpp", + installed: false, + tags: ["chat"], + }, { name: "no-description-model", description: "", @@ -850,7 +916,7 @@ const MARKDOWN_MODELS_RESPONSE = { tags: ["chat"], }, ], - availableModels: 2, + availableModels: 3, installedModels: 0, }; @@ -918,4 +984,65 @@ test.describe("Models Gallery - Markdown descriptions", () => { await expect(row).toBeVisible(); await expect(row.locator("div[title='']")).toHaveText("—"); }); + + test("a heading in the description renders on the UI type scale", async ({ + page, + }) => { + await page.locator("tr", { hasText: "headings-model" }).click(); + const prose = page.locator(".detail-prose__body.markdown-body"); + await expect(prose).toBeVisible(); + + const h1 = prose.locator("h1"); + await expect(h1).toHaveText("Top Heading"); + const sizes = await prose.evaluate((el) => { + const px = (sel) => + parseFloat(getComputedStyle(el.querySelector(sel)).fontSize); + return { h1: px("h1"), h2: px("h2"), p: px("p") }; + }); + // The bug: an unscoped h1 inherits the browser default 2em, which is 26px + // inside this 13px surface and swamps the pane. The scale tops out at + // --text-xl (1.25rem / 20px), so anything at or above that is the default + // leaking through rather than a styled heading. + expect(sizes.h1).toBeGreaterThan(sizes.p); + expect(sizes.h1).toBeLessThanOrEqual(20); + expect(sizes.h1).toBeGreaterThanOrEqual(14); + // The inverse defect: a subheading that is indistinguishable from body + // text. It must stay below h1 and at or above the body size. + expect(sizes.h2).toBeLessThanOrEqual(sizes.h1); + expect(sizes.h2).toBeGreaterThanOrEqual(sizes.p); + }); + + test("the description sits outside the label/value grid on a readable measure", async ({ + page, + }) => { + await page.locator("tr", { hasText: "headings-model" }).click(); + const detail = page.locator('td[colspan="8"]'); + // Description is no longer a row of the scalar table. + await expect(detail.locator("table td", { hasText: "Description" })).toHaveCount( + 0, + ); + await expect(detail.locator(".detail-prose__label")).toHaveText( + "Description", + ); + const proseWidth = await page + .locator(".detail-prose__body") + .evaluate((el) => el.getBoundingClientRect().width); + const paneWidth = await detail.evaluate( + (el) => el.getBoundingClientRect().width, + ); + // A measure, not the full pane: the cap is a ch count, so the exact pixel + // value moves with the font, but it must stay well inside the pane. + expect(proseWidth).toBeLessThan(paneWidth * 0.85); + }); + + test("a model without a description renders no prose block", async ({ + page, + }) => { + await page.locator("tr", { hasText: "no-description-model" }).click(); + const detail = page.locator('td[colspan="8"]'); + await expect(detail).toBeVisible(); + await expect(detail.locator(".detail-prose")).toHaveCount(0); + // The scalar rows still render, so the pane is not blank. + await expect(detail).toContainText("Backend"); + }); }); diff --git a/core/http/react-ui/public/locales/en/models.json b/core/http/react-ui/public/locales/en/models.json index 5bcb7db14785..37d8edfa5fbe 100644 --- a/core/http/react-ui/public/locales/en/models.json +++ b/core/http/react-ui/public/locales/en/models.json @@ -117,10 +117,10 @@ "auto": "Auto", "autoSelected": "Auto-selected", "base": "Base build", - "fits": "Fits", "doesNotFit": "Does not fit", "unknownSize": "Unknown size", "unknownBackend": "Unknown backend", - "loading": "Loading variants..." + "loading": "Loading variants...", + "installVariant": "Install {{variant}}" } } diff --git a/core/http/react-ui/src/App.css b/core/http/react-ui/src/App.css index 130f33084bd5..5ce200b59972 100644 --- a/core/http/react-ui/src/App.css +++ b/core/http/react-ui/src/App.css @@ -9164,3 +9164,233 @@ button.collapsible-header:focus-visible { .model-chip__state { opacity: 0.85; font-style: normal; } .node-filter { margin-bottom: var(--spacing-lg); } .node-detail__metrics { display: flex; gap: var(--spacing-xl); margin: var(--spacing-md) 0 var(--spacing-lg); flex-wrap: wrap; } + +/* Rendered Markdown --------------------------------------------------------- + Gallery descriptions, backend descriptions and voice notes are all + author-supplied Markdown pasted from model cards. Left unscoped they inherit + browser defaults, so a description that opens with `#` renders a 2em heading + inside a 13px surface while a `##` further down is indistinguishable from + body text. This maps the whole element set onto the UI type scale so any + description reads as part of the app rather than as a document dropped into + it. Apply it to every element fed by renderMarkdown(). */ +.markdown-body { + color: var(--color-text-secondary); + font-size: var(--text-sm); + line-height: var(--leading-normal); +} +.markdown-body > :first-child { margin-top: 0; } +.markdown-body > :last-child { margin-bottom: 0; } + +.markdown-body h1, +.markdown-body h2, +.markdown-body h3, +.markdown-body h4, +.markdown-body h5, +.markdown-body h6 { + color: var(--color-text-primary); + font-weight: var(--font-weight-semibold); + line-height: var(--leading-tight); + margin: var(--spacing-md) 0 var(--spacing-xs); +} +.markdown-body h1 { font-size: var(--text-lg); } +.markdown-body h2 { font-size: var(--text-base); } +.markdown-body h3 { font-size: var(--text-sm); } +/* Below h3 the scale has no room left to separate levels by size, so the + remaining depth is carried by case and weight instead. */ +.markdown-body h4, +.markdown-body h5, +.markdown-body h6 { + font-size: var(--text-xs); + color: var(--color-text-secondary); + text-transform: uppercase; + letter-spacing: 0.06em; +} + +.markdown-body p { margin: 0 0 var(--spacing-sm); } +.markdown-body ul, +.markdown-body ol { + margin: 0 0 var(--spacing-sm); + padding-left: var(--spacing-lg); +} +.markdown-body li { margin-bottom: var(--spacing-xs); } +.markdown-body li:last-child { margin-bottom: 0; } +.markdown-body li > ul, +.markdown-body li > ol { margin-top: var(--spacing-xs); } + +.markdown-body a { + color: var(--color-primary); + text-decoration: underline; + text-underline-offset: 2px; +} +.markdown-body a:hover { color: var(--color-primary-hover); } + +.markdown-body code { + font-family: var(--font-mono); + font-size: 0.9em; + background: var(--color-bg-tertiary); + border-radius: var(--radius-sm); + padding: 1px 4px; +} +.markdown-body pre { + margin: 0 0 var(--spacing-sm); + padding: var(--spacing-sm); + background: var(--color-surface-sunken); + border: 1px solid var(--color-border-subtle); + border-radius: var(--radius-md); + overflow-x: auto; +} +.markdown-body pre code { + background: none; + padding: 0; + font-size: var(--text-xs); +} + +.markdown-body blockquote { + margin: 0 0 var(--spacing-sm); + padding-left: var(--spacing-sm); + border-left: 2px solid var(--color-border-strong); + color: var(--color-text-muted); +} +.markdown-body img { + max-width: 100%; + height: auto; + border-radius: var(--radius-sm); +} +.markdown-body hr { + border: 0; + border-top: 1px solid var(--color-border-divider); + margin: var(--spacing-md) 0; +} +/* Tables are the one element that can exceed the measure, so they scroll in + their own box rather than widening the surface around them. */ +.markdown-body table { + display: block; + width: max-content; + max-width: 100%; + overflow-x: auto; + border-collapse: collapse; + margin: 0 0 var(--spacing-sm); + font-size: var(--text-xs); +} +.markdown-body th, +.markdown-body td { + border: 1px solid var(--color-border-subtle); + padding: var(--spacing-xs) var(--spacing-sm); + text-align: left; +} +.markdown-body th { + background: var(--color-bg-tertiary); + font-weight: var(--font-weight-medium); +} + +/* Prose block in a detail pane. The label/value table beside it is right for + scalars, but multi-paragraph prose in a value cell runs the full pane width + and swamps the rows it sits above, so the description gets its own + full-width block with a reading measure. */ +.detail-prose { + margin-bottom: var(--spacing-md); +} +.detail-prose__label { + font-size: var(--text-sm); + font-weight: var(--font-weight-medium); + color: var(--color-text-secondary); + margin-bottom: var(--spacing-xs); +} +.detail-prose__body { + max-width: 68ch; +} + +/* Variant list -------------------------------------------------------------- + One row per alternative build of the same model. Names vary in length, so + the rows share the parent's tracks and line up as columns; the name leads + because it is what the reader is choosing between. Only the informative + status is badged: "fits" is true of nearly every row and says nothing, while + a build that does not fit, and the one that plain Install would pick, do. */ +.variant-list { + /* Content-width rather than pane-width: a full-width row would strand the + install affordance an inch of empty space away from the name it acts on. */ + display: grid; + width: max-content; + max-width: 100%; + grid-template-columns: minmax(0, auto) max-content max-content max-content max-content; + column-gap: var(--spacing-md); + row-gap: 2px; +} +.variant-row { + display: grid; + grid-column: 1 / -1; + grid-template-columns: subgrid; + align-items: center; + width: 100%; + margin: 0; + padding: var(--spacing-xs) var(--spacing-sm); + font: inherit; + text-align: left; + color: inherit; + background: none; + border: 1px solid transparent; + border-radius: var(--radius-sm); + cursor: pointer; + transition: background-color var(--duration-fast) var(--ease-default), + border-color var(--duration-fast) var(--ease-default); +} +/* Subgrid keeps the columns shared across rows; without it each row falls back + to sizing its own tracks, which is ragged but still legible. */ +@supports not (grid-template-columns: subgrid) { + .variant-row { grid-template-columns: minmax(22ch, auto) max-content max-content max-content max-content; } +} +.variant-row:hover:not(:disabled) { + background: var(--color-bg-hover); + border-color: var(--color-border-default); +} +.variant-row:focus-visible { + outline: 2px solid var(--color-focus-ring); + outline-offset: 1px; +} +.variant-row:disabled { + cursor: not-allowed; + opacity: 0.55; +} +.variant-row__name { + font-family: var(--font-mono); + font-size: var(--text-sm); + font-weight: var(--font-weight-medium); + color: var(--color-text-primary); + overflow: hidden; + text-overflow: ellipsis; +} +.variant-row__backend, +.variant-row__size { + font-size: var(--text-xs); + color: var(--color-text-muted); +} +.variant-row__size { + text-align: right; + font-variant-numeric: tabular-nums; +} +.variant-row__status { + display: inline-flex; + align-items: center; + gap: var(--spacing-xs); + flex-wrap: wrap; +} +.variant-row__action { + color: var(--color-text-disabled); + font-size: var(--text-xs); + transition: color var(--duration-fast) var(--ease-default); +} +.variant-row:hover:not(:disabled) .variant-row__action, +.variant-row:focus-visible .variant-row__action { + color: var(--color-primary); +} +/* A build that does not fit stays installable, an explicit choice being an + override the server honours, but it should not read as a peer of the ones + that do. */ +.variant-row--unfit .variant-row__name { + color: var(--color-text-muted); +} + +@media (prefers-reduced-motion: reduce) { + .variant-row, + .variant-row__action { transition: none; } +} diff --git a/core/http/react-ui/src/pages/Backends.jsx b/core/http/react-ui/src/pages/Backends.jsx index deb9385500c2..c19ca9637f01 100644 --- a/core/http/react-ui/src/pages/Backends.jsx +++ b/core/http/react-ui/src/pages/Backends.jsx @@ -848,7 +848,7 @@ function BackendDetail({ backend }) { {backend.description && (
)} diff --git a/core/http/react-ui/src/pages/Manage.jsx b/core/http/react-ui/src/pages/Manage.jsx index 9f612ea779a9..5c871bf6c78d 100644 --- a/core/http/react-ui/src/pages/Manage.jsx +++ b/core/http/react-ui/src/pages/Manage.jsx @@ -1079,7 +1079,7 @@ function ModelDetail({ model, enriched, matchedCaps, distributedMode, onNavigate
{description ? (
) : ( @@ -1189,7 +1189,7 @@ function BackendDetail({ backend, enriched, upgradeInfo, nodes, distributedMode
{description ? (
) : ( diff --git a/core/http/react-ui/src/pages/Models.jsx b/core/http/react-ui/src/pages/Models.jsx index 1f0830bbce3a..664067dbe491 100644 --- a/core/http/react-ui/src/pages/Models.jsx +++ b/core/http/react-ui/src/pages/Models.jsx @@ -672,7 +672,7 @@ export default function Models() { {isExpanded && ( - + )} @@ -796,20 +796,25 @@ function DetailRow({ label, children }) { ) } -function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setExpandedFiles, variantData, t }) { +function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setExpandedFiles, variantData, installing, onInstall, t }) { const files = model.additionalFiles || model.files || [] + const name = model.name || model.id return (
+ {model.description && ( + // Prose sits outside the label/value table: an eight-line value cell + // in a grid of one-line ones breaks the rhythm exactly where the eye + // enters, and the full pane width is roughly double a readable measure. +
+
{t('detail.description')}
+
+
+ )} - - {model.description && ( -
- )} - {model.gallery && ( @@ -848,23 +853,38 @@ function ModelDetail({ model, fit, sizeDisplay, vramDisplay, expandedFiles, setE )} {variantData?.variants?.length > 0 && ( -
- {variantData.variants.map(v => ( -
- {v.model} - {v.backend || t('variants.unknownBackend')} - {variantSizeLabel(v, t)} - - {v.fits ? t('variants.fits') : t('variants.doesNotFit')} - - {v.is_base && {t('variants.base')}} - {v.model === variantData.auto_selected && ( - - {t('variants.autoSelected')} +
+ {variantData.variants.map(v => { + const isAuto = v.model === variantData.auto_selected + return ( + // Listing the alternatives without offering them made the + // detail view read as a menu that could not be ordered + // from; installing one is the same call the split-button + // chevron already makes. +
- ))} +
- +