From c745b96781bc167bea48746bde6c6b3ec985eb26 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:08:06 +0000 Subject: [PATCH] chore(gallery): add Saluki 27B MTP variant Offer the Saluki IQ2 weights with an MTP head for llama.cpp. Pin both assets and document how to select the variant. Assisted-by: Codex:GPT-6 Signed-off-by: Ettore Di Giacinto --- docs/content/features/model-gallery.md | 11 +++++ gallery/index.yaml | 52 ++++++++++++++++++++++ gallery/models/index.yaml | 60 ++++++++++++++++++++++++++ 3 files changed, 123 insertions(+) diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index 56dde2118220..c4bb093a3da3 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -61,6 +61,17 @@ Select a specific build with `--variant minicpm-v-4_7-1b-q4`, `--variant minicpm-v-4_7-1b-q5`, `--variant minicpm-v-4_7-1b-q6`, or `--variant minicpm-v-4_7-1b-q8`. +### Underdog Saluki with multi-token prediction + +The `underdog-saluki-27b-1.0` entry offers `underdog-saluki-27b-1.0-mtp` as a +variant. It preserves the mixed IQ2 main weights and adds an MTP head for +llama.cpp, with up to two draft tokens. Both entries include the F16 vision +projector. To install the MTP build explicitly: + +```bash +local-ai models install --variant underdog-saluki-27b-1.0-mtp underdog-saluki-27b-1.0 +``` + ### Repairing an existing model configuration Gallery template updates do not rewrite YAML files for already installed models. diff --git a/gallery/index.yaml b/gallery/index.yaml index 981cc3563d0d..1cd1b680dfd7 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -3,6 +3,56 @@ # written by scripts/models/cmd/modelcatalog and stays only so releases before # the cutover keep working. CI fails when it differs from the generated copy: # go run ./scripts/models/cmd/modelcatalog --library gallery/models/index.yaml --out gallery/index.yaml +- url: github:mudler/LocalAI/gallery/virtual.yaml@master + name: underdog-saluki-27b-1.0-mtp + description: | + Underdog Saluki 27B 1.0 with its mixed IQ2 main weights preserved and + a Qwen3.8-27B MTP head added. This llama.cpp build enables multi-token + prediction with up to two draft tokens and includes the F16 vision + projector. The weights and projector total approximately 9.28 GB. + license: apache-2.0 + urls: + - https://huggingface.co/Qwen/Qwen3.8-27B + - https://huggingface.co/ConwayResearch/Underdog-Saluki-27B-1.0 + - https://huggingface.co/Kujira/Underdog-Saluki-27B-1.0-MTP-GGUF + tags: + - llm + - gguf + - vision + - function-calling + - reasoning + - mtp + - cpu + - gpu + files: + - filename: llama-cpp/models/underdog-saluki-27b-1.0-mtp/Underdog-Saluki-27B-1.0-IQ2-mix-MTP.gguf + sha256: 98f6ebb527917a2541df8e667d51c3ec8a8bb135b44c606d40cc6ed924e89e52 + uri: https://huggingface.co/Kujira/Underdog-Saluki-27B-1.0-MTP-GGUF/resolve/5d92db9b7aa466a7c8f55e26dac2e366c36e8e5b/Underdog-Saluki-27B-1.0-IQ2-mix-MTP.gguf + - filename: llama-cpp/mmproj/underdog-saluki-27b-1.0/mmproj-Underdog-Saluki-27B-1.0-F16.gguf + sha256: f3aa57813a513420c2ef765503b057c1946bf4c8cdf3679d266a6644fc6cfd50 + uri: https://huggingface.co/ConwayResearch/Underdog-Saluki-27B-1.0/resolve/1336c0b5d74dfe6ad7f793577092f5effd9f97c6/mmproj-Underdog-Saluki-27B-1.0-F16.gguf + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/underdog-saluki-27b-1.0/mmproj-Underdog-Saluki-27B-1.0-F16.gguf + options: + - use_jinja:true + - spec_type:draft-mtp + - spec_n_max:2 + parameters: + model: llama-cpp/models/underdog-saluki-27b-1.0-mtp/Underdog-Saluki-27B-1.0-IQ2-mix-MTP.gguf + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true - url: github:mudler/LocalAI/gallery/virtual.yaml@master name: adavar-3b-q4 description: | @@ -7938,6 +7988,8 @@ top_p: 0.95 template: use_tokenizer_template: true + variants: + - model: underdog-saluki-27b-1.0-mtp - url: github:mudler/LocalAI/gallery/virtual.yaml@master name: spark-x2.5-4b-heretic-jp-q4 description: | diff --git a/gallery/models/index.yaml b/gallery/models/index.yaml index da124ab3a6ca..054f5bc9c242 100644 --- a/gallery/models/index.yaml +++ b/gallery/models/index.yaml @@ -4,6 +4,64 @@ # changes and never edit it by hand. CI fails when it is stale. A build from # master reads it as the default model source; a tagged build reads the gallery # until the release cutover. +- kind: model + id: underdog-saluki-27b-1.0-mtp + name: underdog-saluki-27b-1.0-mtp + version: 0.0.0-gallery.1f1ebd571aaa + description: | + Underdog Saluki 27B 1.0 with its mixed IQ2 main weights preserved and + a Qwen3.8-27B MTP head added. This llama.cpp build enables multi-token + prediction with up to two draft tokens and includes the F16 vision + projector. The weights and projector total approximately 9.28 GB. + license: apache-2.0 + urls: + - https://huggingface.co/Qwen/Qwen3.8-27B + - https://huggingface.co/ConwayResearch/Underdog-Saluki-27B-1.0 + - https://huggingface.co/Kujira/Underdog-Saluki-27B-1.0-MTP-GGUF + tags: + - llm + - gguf + - vision + - function-calling + - reasoning + - mtp + - cpu + - gpu + channel: gallery + policy: + requireSignature: false + schemaVersion: 1 + model: + url: github:mudler/LocalAI/gallery/virtual.yaml@master + files: + - filename: llama-cpp/models/underdog-saluki-27b-1.0-mtp/Underdog-Saluki-27B-1.0-IQ2-mix-MTP.gguf + sha256: 98f6ebb527917a2541df8e667d51c3ec8a8bb135b44c606d40cc6ed924e89e52 + uri: https://huggingface.co/Kujira/Underdog-Saluki-27B-1.0-MTP-GGUF/resolve/5d92db9b7aa466a7c8f55e26dac2e366c36e8e5b/Underdog-Saluki-27B-1.0-IQ2-mix-MTP.gguf + - filename: llama-cpp/mmproj/underdog-saluki-27b-1.0/mmproj-Underdog-Saluki-27B-1.0-F16.gguf + sha256: f3aa57813a513420c2ef765503b057c1946bf4c8cdf3679d266a6644fc6cfd50 + uri: https://huggingface.co/ConwayResearch/Underdog-Saluki-27B-1.0/resolve/1336c0b5d74dfe6ad7f793577092f5effd9f97c6/mmproj-Underdog-Saluki-27B-1.0-F16.gguf + overrides: + backend: llama-cpp + context_size: 32768 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/underdog-saluki-27b-1.0/mmproj-Underdog-Saluki-27B-1.0-F16.gguf + options: + - use_jinja:true + - spec_type:draft-mtp + - spec_n_max:2 + parameters: + model: llama-cpp/models/underdog-saluki-27b-1.0-mtp/Underdog-Saluki-27B-1.0-IQ2-mix-MTP.gguf + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true - kind: model id: adavar-3b-q4 name: adavar-3b-q4 @@ -9187,6 +9245,8 @@ top_p: 0.95 template: use_tokenizer_template: true + variants: + - model: underdog-saluki-27b-1.0-mtp - kind: model id: spark-x2.5-4b-heretic-jp-q4 name: spark-x2.5-4b-heretic-jp-q4