fix(model-artifacts): materialize longcat-video on the controller, and support companion repos

backend/python/longcat-video/backend.py

@@ -109,6 +109,7 @@ def __init__(self):
         self.device_index = 0
         self.cp_split_hw = None
         self._dist_store_dir = None
+        self.model_path = ""
 
     def Health(self, request, context):
         return backend_pb2.Reply(message=b"OK")
@@ -118,6 +119,11 @@ def LoadModel(self, request, context):
         if request.ModelFile and os.path.isdir(request.ModelFile):
             model = request.ModelFile
 
+        # A managed companion snapshot (see base_model) is passed as a path
+        # relative to the models directory, which is ModelPath here and a
+        # different directory on a remote worker once staging has rewritten it.
+        self.model_path = getattr(request, "ModelPath", "") or ""
+
         model_kind = classify_model(model)
         if model_kind is None:
             return self._fail(
@@ -376,6 +382,26 @@ def _ensure_distributed(self):
             )
             self.cp_split_hw = self.context_parallel_util.get_optimal_split(1)
 
+    def _resolve_option_path(self, value):
+        """Resolve an option that may name a local directory or a HF repo id.
+
+        A managed companion artifact is handed to us relative to the models
+        directory, so it only becomes a real path once joined with ModelPath;
+        that indirection is what lets the same config work on a remote worker,
+        where staging puts the snapshot somewhere else entirely. Anything that
+        is already an absolute directory, or is not a path at all (a plain repo
+        id), is passed through untouched for snapshot_download to handle.
+        """
+        if not value:
+            return value
+        candidate = normalize_model_source(str(value))
+        if os.path.isabs(candidate) or not self.model_path:
+            return value
+        joined = os.path.join(self.model_path, candidate)
+        if os.path.isdir(joined):
+            return joined
+        return value
+
     def _resolve_checkpoint(self, model, patterns):
         source = normalize_model_source(model)
         if os.path.isdir(source):
@@ -440,13 +466,7 @@ def _load_avatar_model(self, model):
         avatar_patterns.append(f"{model_subfolder}/**")
         checkpoint = self._resolve_checkpoint(model, avatar_patterns)
 
-        base_model = self.options.get("base_model")
-        if not base_model and os.path.isdir(normalize_model_source(model)):
-            sibling = os.path.join(
-                os.path.dirname(normalize_model_source(model)), "LongCat-Video"
-            )
-            if os.path.isdir(sibling):
-                base_model = sibling
+        base_model = self._resolve_option_path(self.options.get("base_model"))
         base_model = base_model or BASE_MODEL_ID
         if classify_model(str(base_model)) != MODEL_KIND_BASE:
             raise ValueError("base_model must point to a LongCat-Video checkpoint")

core/backend/companion_options_test.go

@@ -0,0 +1,123 @@
+package backend
+
+import (
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/pkg/modelartifacts"
+)
+
+// A companion snapshot only becomes useful once the backend can find it, and
+// its location is a content-addressed cache key that is unknowable until the
+// artifact resolves. A static gallery override therefore cannot carry it. The
+// path is instead synthesized into ModelOptions.Options at load time, under the
+// companion's own name, using the same key:value convention backends already
+// parse for options like attention_backend.
+var _ = Describe("companion artifact backend options", func() {
+	const (
+		primaryKey   = "1111111111111111111111111111111111111111111111111111111111111111"
+		companionKey = "2222222222222222222222222222222222222222222222222222222222222222"
+	)
+
+	resolved := func(key string) *modelartifacts.Resolved {
+		return &modelartifacts.Resolved{
+			Endpoint: "https://huggingface.co",
+			Revision: "0123456789abcdef0123456789abcdef01234567",
+			CacheKey: key,
+		}
+	}
+
+	threads := 1
+	configWithCompanion := func(options ...string) config.ModelConfig {
+		return config.ModelConfig{
+			Backend: "longcat-video",
+			Options: options,
+			Threads: &threads,
+			Artifacts: []modelartifacts.Spec{
+				{
+					Name: modelartifacts.TargetModel, Target: modelartifacts.TargetModel,
+					Source:   modelartifacts.Source{Type: "huggingface", Repo: "meituan-longcat/LongCat-Video-Avatar-1.5", Revision: "main"},
+					Resolved: resolved(primaryKey),
+				},
+				{
+					Name: "base_model", Target: modelartifacts.TargetCompanion,
+					Source:   modelartifacts.Source{Type: "huggingface", Repo: "meituan-longcat/LongCat-Video", Revision: "main"},
+					Resolved: resolved(companionKey),
+				},
+			},
+		}
+	}
+
+	optionValue := func(options []string, key string) (string, bool) {
+		for _, option := range options {
+			name, value, found := strings.Cut(option, ":")
+			if found && name == key {
+				return value, true
+			}
+		}
+		return "", false
+	}
+
+	It("exposes a resolved companion snapshot as an option named after the artifact", func() {
+		opts := grpcModelOpts(configWithCompanion("attention_backend:sdpa"), "/models")
+
+		value, found := optionValue(opts.Options, "base_model")
+		Expect(found).To(BeTrue())
+		expected, err := modelartifacts.RelativeSnapshotPath(companionKey)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(value).To(Equal(expected))
+		// The companion path must never be confused with the load target.
+		Expect(value).ToNot(ContainSubstring(primaryKey))
+		// Author-supplied options survive untouched.
+		Expect(opts.Options).To(ContainElement("attention_backend:sdpa"))
+	})
+
+	It("keeps the path relative so the worker resolves it under its own ModelPath", func() {
+		opts := grpcModelOpts(configWithCompanion(), "/models")
+
+		value, found := optionValue(opts.Options, "base_model")
+		Expect(found).To(BeTrue())
+		Expect(strings.HasPrefix(value, "/")).To(BeFalse())
+	})
+
+	It("does not override an explicitly configured option of the same name", func() {
+		// An operator pinning base_model to a local checkout must win over the
+		// synthesized value.
+		opts := grpcModelOpts(configWithCompanion("base_model:/opt/checkouts/LongCat-Video"), "/models")
+
+		Expect(opts.Options).To(ContainElement("base_model:/opt/checkouts/LongCat-Video"))
+		values := 0
+		for _, option := range opts.Options {
+			if strings.HasPrefix(option, "base_model:") {
+				values++
+			}
+		}
+		Expect(values).To(Equal(1))
+	})
+
+	It("synthesizes nothing for a config without companions", func() {
+		cfg := config.ModelConfig{
+			Backend: "transformers",
+			Options: []string{"attention_backend:sdpa"},
+			Threads: &threads,
+			Artifacts: []modelartifacts.Spec{{
+				Name: modelartifacts.TargetModel, Target: modelartifacts.TargetModel,
+				Source:   modelartifacts.Source{Type: "huggingface", Repo: "owner/repo", Revision: "main"},
+				Resolved: resolved(primaryKey),
+			}},
+		}
+		opts := grpcModelOpts(cfg, "/models")
+		Expect(opts.Options).To(Equal([]string{"attention_backend:sdpa"}))
+	})
+
+	It("skips a companion that has not been resolved yet", func() {
+		cfg := configWithCompanion()
+		cfg.Artifacts[1].Resolved = nil
+		opts := grpcModelOpts(cfg, "/models")
+		_, found := optionValue(opts.Options, "base_model")
+		Expect(found).To(BeFalse())
+	})
+})

core/backend/options.go

@@ -7,6 +7,7 @@ import (
 	"math/rand/v2"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"time"
 
@@ -280,6 +281,48 @@ func EffectiveBatchSize(c config.ModelConfig) int {
 	return DefaultBatchSize
 }
 
+// withCompanionArtifactOptions surfaces each resolved companion snapshot to the
+// backend as "<artifact name>:<snapshot path>", reusing the key:value option
+// convention backends already parse.
+//
+// The value is deliberately relative to the models directory and deliberately
+// not persisted to the config YAML. It is derived from a content-addressed cache
+// key that only exists after the artifact resolves, so a static gallery override
+// could not carry it, and a persisted copy would rot the moment a re-resolve
+// produced a new key. Staying relative also lets a remote worker resolve it
+// under its own ModelPath after staging rewrites the model root.
+//
+// An option the author set explicitly always wins: pinning a companion to a
+// local checkout has to beat the managed snapshot.
+func withCompanionArtifactOptions(options []string, artifacts []modelartifacts.Spec) []string {
+	configured := make(map[string]struct{}, len(options))
+	for _, option := range options {
+		if name, _, found := strings.Cut(option, ":"); found {
+			configured[name] = struct{}{}
+		}
+	}
+
+	// Copy before appending: opts.Options would otherwise share (and could
+	// reallocate away from) the config's own slice.
+	combined := slices.Clone(options)
+	for _, artifact := range artifacts {
+		if artifact.Target != modelartifacts.TargetCompanion || artifact.Resolved == nil {
+			continue
+		}
+		if _, exists := configured[artifact.Name]; exists {
+			xlog.Debug("keeping the configured companion option over the managed snapshot", "artifact", artifact.Name)
+			continue
+		}
+		snapshot, err := modelartifacts.RelativeSnapshotPath(artifact.Resolved.CacheKey)
+		if err != nil {
+			xlog.Warn("skipping companion artifact with an unusable cache key", "artifact", artifact.Name, "error", err)
+			continue
+		}
+		combined = append(combined, artifact.Name+":"+snapshot)
+	}
+	return combined
+}
+
 func grpcModelOpts(c config.ModelConfig, modelPath string) *pb.ModelOptions {
 	ctxSize := EffectiveContextSize(c)
 	b := EffectiveBatchSize(c)
@@ -370,7 +413,7 @@ func grpcModelOpts(c config.ModelConfig, modelPath string) *pb.ModelOptions {
 		IMG2IMG:              c.Diffusers.IMG2IMG,
 		CLIPModel:            c.Diffusers.ClipModel,
 		CLIPSubfolder:        c.Diffusers.ClipSubFolder,
-		Options:              c.Options,
+		Options:              withCompanionArtifactOptions(c.Options, c.Artifacts),
 		Overrides:            c.Overrides,
 		EngineArgs:           engineArgsJSON,
 		CLIPSkip:             int32(c.Diffusers.ClipSkip),

core/config/model_artifact_companion_test.go

@@ -0,0 +1,113 @@
+package config_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+
+	"github.com/mudler/LocalAI/core/config"
+)
+
+// A composed pipeline needs more than one snapshot: LongCat-Video-Avatar-1.5
+// loads its own transformer but takes tokenizer, text encoder and VAE from the
+// LongCat-Video base repo. The config expresses that as one target: model
+// artifact followed by named target: companion artifacts. Exactly one primary,
+// and it must be first, because ModelFileName() and every load path resolve the
+// load target from Artifacts[0].
+var _ = Describe("multi-artifact config validation", func() {
+	parse := func(doc string) config.ModelConfig {
+		var c config.ModelConfig
+		Expect(yaml.Unmarshal([]byte(doc), &c)).To(Succeed())
+		return c
+	}
+
+	It("accepts a primary followed by a named companion", func() {
+		c := parse(`
+backend: longcat-video
+artifacts:
+  - name: model
+    target: model
+    source: {type: huggingface, repo: meituan-longcat/LongCat-Video-Avatar-1.5}
+  - name: base_model
+    target: companion
+    source: {type: huggingface, repo: meituan-longcat/LongCat-Video}
+parameters: {model: meituan-longcat/LongCat-Video-Avatar-1.5}
+`)
+		valid, err := c.Validate()
+		Expect(err).NotTo(HaveOccurred())
+		Expect(valid).To(BeTrue())
+	})
+
+	It("rejects a config whose artifacts are all companions", func() {
+		// The reachable way to end up without a primary. Two primaries cannot
+		// coexist by construction: both would have to be named "model", so the
+		// duplicate-name rule catches that case first (covered below).
+		c := parse(`
+backend: longcat-video
+artifacts:
+  - name: base_model
+    target: companion
+    source: {type: huggingface, repo: owner/base}
+parameters: {model: owner/main}
+`)
+		valid, err := c.Validate()
+		Expect(valid).To(BeFalse())
+		Expect(err).To(MatchError(ContainSubstring("exactly one")))
+	})
+
+	It("rejects a second artifact claiming the primary target", func() {
+		c := parse(`
+backend: longcat-video
+artifacts:
+  - name: model
+    target: model
+    source: {type: huggingface, repo: owner/one}
+  - name: second
+    target: model
+    source: {type: huggingface, repo: owner/two}
+parameters: {model: owner/one}
+`)
+		valid, err := c.Validate()
+		Expect(valid).To(BeFalse())
+		Expect(err).To(MatchError(ContainSubstring("primary artifact name")))
+	})
+
+	It("rejects a companion declared before the primary", func() {
+		// Artifacts[0] is the load target everywhere; a companion in that slot
+		// would silently point the backend at the wrong snapshot.
+		c := parse(`
+backend: longcat-video
+artifacts:
+  - name: base_model
+    target: companion
+    source: {type: huggingface, repo: owner/base}
+  - name: model
+    target: model
+    source: {type: huggingface, repo: owner/main}
+parameters: {model: owner/main}
+`)
+		valid, err := c.Validate()
+		Expect(valid).To(BeFalse())
+		Expect(err).To(MatchError(ContainSubstring("first")))
+	})
+
+	It("rejects companions sharing a name", func() {
+		c := parse(`
+backend: longcat-video
+artifacts:
+  - name: model
+    target: model
+    source: {type: huggingface, repo: owner/main}
+  - name: base_model
+    target: companion
+    source: {type: huggingface, repo: owner/base}
+  - name: base_model
+    target: companion
+    source: {type: huggingface, repo: owner/other}
+parameters: {model: owner/main}
+`)
+		valid, err := c.Validate()
+		Expect(valid).To(BeFalse())
+		Expect(err).To(MatchError(ContainSubstring("duplicate")))
+	})
+})

core/config/model_artifact_inference.go

@@ -21,6 +21,7 @@ var managedArtifactBackends = map[string]struct{}{
 	"transformers-musicgen": {}, "mamba": {}, "diffusers": {}, "qwen-asr": {},
 	"fish-speech": {}, "nemo": {}, "voxcpm": {}, "qwen-tts": {},
 	"liquid-audio": {}, "vllm": {}, "vllm-omni": {}, "sglang": {},
+	"longcat-video": {},
 }
 
 // IsManagedArtifactBackend reports whether backend consumes a model as a

core/config/model_artifact_inference_test.go

@@ -42,6 +42,20 @@ var _ = Describe("PrimaryArtifactSpec backend gating", func() {
 		Expect(spec.Source.Repo).To(Equal("owner/repo"))
 	})
 
+	It("infers a managed artifact for longcat-video from a bare repo id", func() {
+		// longcat-video loads a checkpoint directory (its backend.py takes
+		// request.ModelFile when os.path.isdir), so it belongs to the same
+		// class as transformers/vllm/diffusers. Off the allow-list, the
+		// controller never acquires the weights and the backend re-downloads
+		// them from HuggingFace inside the remote LoadModel deadline.
+		c := parse("backend: longcat-video\nparameters: {model: meituan-longcat/LongCat-Video-Avatar-1.5}\n")
+		spec, inferred, managed, err := c.PrimaryArtifactSpec("/models")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(managed).To(BeTrue())
+		Expect(inferred).To(BeTrue())
+		Expect(spec.Source.Repo).To(Equal("meituan-longcat/LongCat-Video-Avatar-1.5"))
+	})
+
 	It("keeps explicit artifacts managed even on a single-file backend", func() {
 		// An explicitly declared artifacts: block is a deliberate choice;
 		// single-file resolution (PrimaryFile) handles the load path.