fix(distributed): backend discovery hid GPU-only backends behind the controller's capability

core/gallery/backends_nodecapability_test.go

@@ -0,0 +1,131 @@
+package gallery_test
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/mudler/LocalAI/core/config"
+	. "github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/pkg/system"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+)
+
+// These specs cover backend discovery in distributed mode, where the machine
+// answering GET /backends/available (the controller) is not the machine that
+// will run the backend (a worker). Filtering the listing against the
+// controller's own hardware hid every GPU-only meta backend from admins.
+var _ = Describe("AvailableBackendsForCapabilities", func() {
+	var (
+		tempDir     string
+		galleryPath string
+		galleries   []config.Gallery
+		// controller stands in for a GPU-less frontend pod: no vendor, so its
+		// reported capability is "default".
+		controller *system.SystemState
+	)
+
+	writeGalleryYAML := func(backends []GalleryBackend) {
+		data, err := yaml.Marshal(backends)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(os.WriteFile(galleryPath, data, 0644)).To(Succeed())
+	}
+
+	names := func(backends GalleryElements[*GalleryBackend]) []string {
+		out := []string{}
+		for _, b := range backends {
+			out = append(out, b.GetName())
+		}
+		return out
+	}
+
+	// longcatVideo mirrors the real gallery entry that triggered this bug: a
+	// meta backend enumerating only NVIDIA variants, with neither a "default"
+	// nor a "cpu" key for Capability() to fall back to.
+	longcatVideo := GalleryBackend{
+		Metadata: Metadata{Name: "longcat-video"},
+		CapabilitiesMap: map[string]string{
+			"nvidia":             "longcat-video-nvidia",
+			"nvidia-cuda-12":     "longcat-video-cuda-12",
+			"nvidia-cuda-13":     "longcat-video-cuda-13",
+			"nvidia-l4t-cuda-13": "longcat-video-l4t-cuda-13",
+		},
+	}
+
+	// cpuMeta is compatible with every host and must keep showing up in all
+	// scenarios, proving the union widens the listing without replacing it.
+	cpuMeta := GalleryBackend{
+		Metadata:        Metadata{Name: "whisper"},
+		CapabilitiesMap: map[string]string{"default": "whisper-cpu", "nvidia": "whisper-cuda"},
+	}
+
+	BeforeEach(func() {
+		var err error
+		tempDir, err = os.MkdirTemp("", "node-capability-test-*")
+		Expect(err).NotTo(HaveOccurred())
+		DeferCleanup(func() {
+			Expect(os.RemoveAll(tempDir)).To(Succeed())
+		})
+
+		galleryPath = filepath.Join(tempDir, "gallery.yaml")
+		writeGalleryYAML([]GalleryBackend{longcatVideo, cpuMeta})
+
+		galleries = []config.Gallery{{Name: "test-gallery", URL: "file://" + galleryPath}}
+		controller = system.NewCapabilityState("default", system.WithBackendPath(tempDir))
+	})
+
+	It("hides a GPU-only meta when no node capabilities are supplied", func() {
+		backends, err := AvailableBackendsForCapabilities(galleries, controller, nil)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(names(backends)).To(ContainElement("whisper"))
+		Expect(names(backends)).NotTo(ContainElement("longcat-video"))
+	})
+
+	It("lists a GPU-only meta runnable on a registered worker node", func() {
+		backends, err := AvailableBackendsForCapabilities(galleries, controller, []string{"nvidia-l4t-cuda-13"})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(names(backends)).To(ContainElement("longcat-video"))
+		Expect(names(backends)).To(ContainElement("whisper"))
+	})
+
+	It("unions across heterogeneous nodes rather than intersecting them", func() {
+		writeGalleryYAML([]GalleryBackend{
+			longcatVideo,
+			{
+				Metadata:        Metadata{Name: "amd-only"},
+				CapabilitiesMap: map[string]string{"amd": "amd-only-rocm"},
+			},
+		})
+
+		backends, err := AvailableBackendsForCapabilities(galleries, controller, []string{"nvidia", "amd"})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(names(backends)).To(ContainElements("longcat-video", "amd-only"))
+	})
+
+	It("still excludes a meta no node can satisfy", func() {
+		backends, err := AvailableBackendsForCapabilities(galleries, controller, []string{"amd"})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(names(backends)).NotTo(ContainElement("longcat-video"))
+	})
+
+	It("filters concrete backends by node capability too", func() {
+		writeGalleryYAML([]GalleryBackend{
+			{Metadata: Metadata{Name: "some-backend-cuda"}, URI: "quay.io/test/cuda"},
+			{Metadata: Metadata{Name: "some-backend-rocm"}, URI: "quay.io/test/rocm"},
+		})
+
+		backends, err := AvailableBackendsForCapabilities(galleries, controller, []string{"nvidia"})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(names(backends)).To(ContainElement("some-backend-cuda"))
+		Expect(names(backends)).NotTo(ContainElement("some-backend-rocm"))
+	})
+
+	It("returns exactly the single-node listing when the node list is empty", func() {
+		withNodes, err := AvailableBackendsForCapabilities(galleries, controller, []string{})
+		Expect(err).NotTo(HaveOccurred())
+		baseline, err := AvailableBackends(galleries, controller)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(names(withNodes)).To(Equal(names(baseline)))
+	})
+})

core/gallery/gallery.go

@@ -395,16 +395,57 @@ func triggerGalleryRefresh(galleries []config.Gallery, systemState *system.Syste
 
 // List available backends
 func AvailableBackends(galleries []config.Gallery, systemState *system.SystemState) (GalleryElements[*GalleryBackend], error) {
-	return availableBackendsWithFilter(galleries, systemState, true)
+	return availableBackendsWithFilter(galleries, systemState, func(backend *GalleryBackend) bool {
+		return backend.IsCompatibleWith(systemState)
+	})
 }
 
 // AvailableBackendsUnfiltered returns all available backends without filtering by system capability.
 func AvailableBackendsUnfiltered(galleries []config.Gallery, systemState *system.SystemState) (GalleryElements[*GalleryBackend], error) {
-	return availableBackendsWithFilter(galleries, systemState, false)
+	return availableBackendsWithFilter(galleries, systemState, nil)
 }
 
-// availableBackendsWithFilter is a helper function that lists available backends with optional filtering.
-func availableBackendsWithFilter(galleries []config.Gallery, systemState *system.SystemState, filterByCapability bool) (GalleryElements[*GalleryBackend], error) {
+// AvailableBackendsForCapabilities lists backends runnable on the local system
+// OR on any remote host reporting one of the supplied capabilities.
+//
+// In a distributed deployment the host serving this listing (the controller)
+// is usually a GPU-less pod while the GPUs live on worker nodes. Filtering
+// only against the controller hid every GPU-only meta backend from admins even
+// though installing it by name on a worker worked fine, so compatibility is
+// evaluated as a union over the cluster. An empty capabilities slice reproduces
+// AvailableBackends exactly, keeping single-node behavior untouched.
+func AvailableBackendsForCapabilities(galleries []config.Gallery, systemState *system.SystemState, capabilities []string) (GalleryElements[*GalleryBackend], error) {
+	if len(capabilities) == 0 {
+		return AvailableBackends(galleries, systemState)
+	}
+
+	// Each remote capability is evaluated through a state pinned to that exact
+	// capability, so the controller's own detection (and any forced capability
+	// on the controller image) cannot leak into the worker's verdict. Backend
+	// paths still come from the controller's state because that is where the
+	// gallery metadata is read from.
+	nodeStates := make([]*system.SystemState, 0, len(capabilities))
+	for _, capability := range capabilities {
+		nodeStates = append(nodeStates, system.NewCapabilityState(capability,
+			system.WithBackendPath(systemState.Backend.BackendsPath)))
+	}
+
+	return availableBackendsWithFilter(galleries, systemState, func(backend *GalleryBackend) bool {
+		if backend.IsCompatibleWith(systemState) {
+			return true
+		}
+		for _, nodeState := range nodeStates {
+			if backend.IsCompatibleWith(nodeState) {
+				return true
+			}
+		}
+		return false
+	})
+}
+
+// availableBackendsWithFilter lists available backends, keeping only those
+// accepted by compatible. A nil compatible keeps everything.
+func availableBackendsWithFilter(galleries []config.Gallery, systemState *system.SystemState, compatible func(*GalleryBackend) bool) (GalleryElements[*GalleryBackend], error) {
 	var backends []*GalleryBackend
 
 	systemBackends, err := ListSystemBackends(systemState)
@@ -421,15 +462,15 @@ func availableBackendsWithFilter(galleries []config.Gallery, systemState *system
 			return nil, err
 		}
 
-		// Filter backends by system capability if requested
-		if filterByCapability {
-			for _, backend := range galleryBackends {
-				if backend.IsCompatibleWith(systemState) {
-					backends = append(backends, backend)
-				}
-			}
-		} else {
+		if compatible == nil {
 			backends = append(backends, galleryBackends...)
+			continue
+		}
+
+		for _, backend := range galleryBackends {
+			if compatible(backend) {
+				backends = append(backends, backend)
+			}
 		}
 	}
 

core/http/app.go

@@ -443,7 +443,7 @@ func API(application *application.Application) (*echo.Echo, error) {
 		ftNats,
 		ftStore,
 	)
-	routes.RegisterFineTuningRoutes(e, ftService, application.ApplicationConfig(), fineTuningMw)
+	routes.RegisterFineTuningRoutes(e, ftService, application.ApplicationConfig(), application, fineTuningMw)
 
 	// Quantization routes
 	quantizationMw := auth.RequireFeature(application.AuthDB(), auth.FeatureQuantization)
@@ -465,7 +465,7 @@ func API(application *application.Application) (*echo.Echo, error) {
 		quantNats,
 		quantStore,
 	)
-	routes.RegisterQuantizationRoutes(e, qService, application.ApplicationConfig(), quantizationMw)
+	routes.RegisterQuantizationRoutes(e, qService, application.ApplicationConfig(), application, quantizationMw)
 
 	// Node management routes (distributed mode)
 	distCfg := application.ApplicationConfig().Distributed

core/http/endpoints/localai/backend.go

@@ -1,6 +1,7 @@
 package localai
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"sort"
@@ -323,14 +324,41 @@ func (mgs *BackendEndpointService) UpgradeBackendEndpoint() echo.HandlerFunc {
 	}
 }
 
+// ClusterCapabilityProvider reports the meta-backend capabilities available
+// somewhere in the cluster. It is nil in single-node deployments, where the
+// local system state is the only thing worth filtering against.
+type ClusterCapabilityProvider func(ctx context.Context) ([]string, error)
+
+// resolveClusterCapabilities reads the capabilities present in the cluster,
+// degrading to the local-only listing on error.
+//
+// Every capability-filtered discovery endpoint shares this: on a distributed
+// controller the GPUs live on the workers, so filtering against the local
+// (usually GPU-less) host hides GPU-only backends the cluster can actually
+// run. A registry hiccup must never blank the catalog, so a failure falls back
+// to the pre-existing local-only behavior rather than erroring the request.
+func resolveClusterCapabilities(ctx context.Context, provider ClusterCapabilityProvider) []string {
+	if provider == nil {
+		return nil
+	}
+	capabilities, err := provider(ctx)
+	if err != nil {
+		xlog.Warn("Could not read cluster capabilities, listing backends for the local system only", "error", err)
+		return nil
+	}
+	return capabilities
+}
+
 // ListAvailableBackendsEndpoint list the available backends in the galleries configured in LocalAI
 // @Summary List all available Backends
 // @Tags backends
 // @Success 200 {object} []gallery.GalleryBackend "Response"
 // @Router /backends/available [get]
-func (mgs *BackendEndpointService) ListAvailableBackendsEndpoint(systemState *system.SystemState) echo.HandlerFunc {
+func (mgs *BackendEndpointService) ListAvailableBackendsEndpoint(systemState *system.SystemState, clusterCapabilities ClusterCapabilityProvider) echo.HandlerFunc {
 	return func(c echo.Context) error {
-		backends, err := gallery.AvailableBackends(mgs.galleries, systemState)
+		capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
+
+		backends, err := gallery.AvailableBackendsForCapabilities(mgs.galleries, systemState, capabilities)
 		if err != nil {
 			return err
 		}

core/http/endpoints/localai/backend_available_cluster_test.go

@@ -0,0 +1,109 @@
+package localai_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/gallery"
+	. "github.com/mudler/LocalAI/core/http/endpoints/localai"
+	"github.com/mudler/LocalAI/pkg/system"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+)
+
+// GET /backends/available on a distributed controller must advertise what the
+// cluster can run, not what the controller itself can run.
+var _ = Describe("ListAvailableBackendsEndpoint cluster capabilities", func() {
+	var (
+		app         *echo.Echo
+		systemState *system.SystemState
+		tmpDir      string
+		galleries   []config.Gallery
+	)
+
+	BeforeEach(func() {
+		app = echo.New()
+
+		var err error
+		tmpDir, err = os.MkdirTemp("", "backends-available-cluster-*")
+		Expect(err).NotTo(HaveOccurred())
+		DeferCleanup(func() {
+			Expect(os.RemoveAll(tmpDir)).To(Succeed())
+		})
+
+		galleryPath := filepath.Join(tmpDir, "gallery.yaml")
+		data, err := yaml.Marshal([]gallery.GalleryBackend{
+			{
+				Metadata: gallery.Metadata{Name: "longcat-video"},
+				CapabilitiesMap: map[string]string{
+					"nvidia":             "longcat-video-nvidia",
+					"nvidia-cuda-13":     "longcat-video-cuda-13",
+					"nvidia-l4t-cuda-13": "longcat-video-l4t-cuda-13",
+				},
+			},
+			{
+				Metadata:        gallery.Metadata{Name: "whisper"},
+				CapabilitiesMap: map[string]string{"default": "whisper-cpu"},
+			},
+		})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(os.WriteFile(galleryPath, data, 0o644)).To(Succeed())
+
+		galleries = []config.Gallery{{Name: "test-gallery", URL: "file://" + galleryPath}}
+
+		// A GPU-less controller pod: capability "default".
+		systemState = system.NewCapabilityState("default",
+			system.WithBackendPath(tmpDir), system.WithBackendSystemPath(tmpDir))
+	})
+
+	listNames := func(provider ClusterCapabilityProvider) []string {
+		svc := CreateBackendEndpointService(galleries, systemState, nil, nil)
+		app.GET("/backends/available", svc.ListAvailableBackendsEndpoint(systemState, provider))
+
+		req := httptest.NewRequest(http.MethodGet, "/backends/available", nil)
+		rec := httptest.NewRecorder()
+		app.ServeHTTP(rec, req)
+		Expect(rec.Code).To(Equal(http.StatusOK))
+
+		var backends []gallery.GalleryBackend
+		Expect(json.Unmarshal(rec.Body.Bytes(), &backends)).To(Succeed())
+
+		names := []string{}
+		for _, b := range backends {
+			names = append(names, b.Name)
+		}
+		return names
+	}
+
+	It("hides GPU-only backends with no provider (single-node)", func() {
+		names := listNames(nil)
+		Expect(names).To(ContainElement("whisper"))
+		Expect(names).NotTo(ContainElement("longcat-video"))
+	})
+
+	It("lists GPU-only backends a worker node can run", func() {
+		provider := func(ctx context.Context) ([]string, error) {
+			return []string{"nvidia-l4t-cuda-13"}, nil
+		}
+		Expect(listNames(provider)).To(ContainElements("whisper", "longcat-video"))
+	})
+
+	It("falls back to the local listing when the registry errors", func() {
+		// A registry hiccup must degrade to the old behavior, never blank the
+		// backend catalog with a 500.
+		provider := func(ctx context.Context) ([]string, error) {
+			return nil, errors.New("registry unavailable")
+		}
+		names := listNames(provider)
+		Expect(names).To(ContainElement("whisper"))
+		Expect(names).NotTo(ContainElement("longcat-video"))
+	})
+})