feat(classifier/VAD): support voice control on low power devices

.docker/llama-cpp-compile.sh

@@ -28,6 +28,10 @@ if [ -z "${BUILD_TYPE:-}" ]; then
   # variants with it (the host never *selects* SME unless it has it, but every variant must
   # still compile).
   if [ "${TARGETARCH}" = "arm64" ]; then
+    # The prebuilt base inherits default ports.ubuntu.com sources; honor the
+    # APT_*_MIRROR build args here like the from-source path does, so this
+    # apt step survives a mirror outage.
+    sh /LocalAI/.docker/apt-mirror.sh || true
     apt-get update -qq && apt-get install -y -qq gcc-14 g++-14
     export CC=gcc-14 CXX=g++-14
   fi

.github/backend-matrix.yml

@@ -478,6 +478,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "12"
+    cuda-minor-version: "8"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-12-longcat-video'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "longcat-video"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "12"
     cuda-minor-version: "8"
@@ -1149,6 +1162,19 @@ include:
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'cublas'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-nvidia-cuda-13-longcat-video'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "longcat-video"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'cublas'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -1357,6 +1383,19 @@ include:
     backend: "vllm-omni"
     dockerfile: "./backend/Dockerfile.python"
     context: "./"
+  - build-type: 'l4t'
+    cuda-major-version: "13"
+    cuda-minor-version: "0"
+    platforms: 'linux/arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-nvidia-l4t-cuda-13-arm64-longcat-video'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    ubuntu-version: '2404'
+    backend: "longcat-video"
+    dockerfile: "./backend/Dockerfile.python"
+    context: "./"
   - build-type: 'l4t'
     cuda-major-version: "13"
     cuda-minor-version: "0"
@@ -4800,7 +4839,6 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
-  # rfdetr
   - build-type: ''
     cuda-major-version: ""
     cuda-minor-version: ""
@@ -4815,6 +4853,35 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  # cloud-proxy
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-cloud-proxy'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cloud-proxy"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-cloud-proxy'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cloud-proxy"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   # rfdetr
   - build-type: ''
     cuda-major-version: ""
@@ -5517,6 +5584,10 @@ includeDarwin:
     tag-suffix: "-metal-darwin-arm64-local-store"
     build-type: "metal"
     lang: "go"
+  - backend: "cloud-proxy"
+    tag-suffix: "-metal-darwin-arm64-cloud-proxy"
+    build-type: "metal"
+    lang: "go"
   - backend: "llama-cpp-quantization"
     tag-suffix: "-metal-darwin-arm64-llama-cpp-quantization"
     build-type: "mps"

.github/workflows/bump_deps.yaml

@@ -26,6 +26,10 @@ jobs:
             variable: "DS4_VERSION"
             branch: "main"
             file: "backend/cpp/ds4/Makefile"
+          - repository: "meituan-longcat/LongCat-Video"
+            variable: "LONGCAT_VIDEO_VERSION"
+            branch: "main"
+            file: "backend/python/longcat-video/Makefile"
           - repository: "localai-org/privacy-filter.cpp"
             variable: "PRIVACY_FILTER_VERSION"
             branch: "master"

.github/workflows/stalebot.yml

@@ -11,7 +11,7 @@ jobs:
     if: github.repository == 'mudler/LocalAI'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v9
+      - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v9
         with:
           stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 5 days.'
           stale-pr-message: 'This PR is stale because it has been open 90 days with no activity. Remove stale label or comment or this will be closed in 10 days.'

.impeccable/live/config.json

@@ -0,0 +1,6 @@
+{
+  "files": ["core/http/react-ui/index.html"],
+  "insertBefore": "</body>",
+  "commentSyntax": "html",
+  "cspChecked": true
+}

Makefile

@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/outetts backends/piper backends/stablediffusion-ggml backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin
 
 GOCMD=go
 GOTEST=$(GOCMD) test
@@ -565,6 +565,7 @@ prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/chatterbox
 	$(MAKE) -C backend/python/vllm
 	$(MAKE) -C backend/python/vllm-omni
+	$(MAKE) -C backend/python/longcat-video
 	$(MAKE) -C backend/python/sglang
 	$(MAKE) -C backend/python/vibevoice
 	$(MAKE) -C backend/python/liquid-audio
@@ -594,6 +595,7 @@ test-extra: prepare-test-extra
 	$(MAKE) -C backend/python/chatterbox test
 	$(MAKE) -C backend/python/vllm test
 	$(MAKE) -C backend/python/vllm-omni test
+	$(MAKE) -C backend/python/longcat-video test
 	$(MAKE) -C backend/python/vibevoice test
 	$(MAKE) -C backend/python/liquid-audio test
 	$(MAKE) -C backend/python/moonshine test
@@ -1254,6 +1256,7 @@ BACKEND_NEUTTS = neutts|python|.|false|true
 BACKEND_KOKORO = kokoro|python|.|false|true
 BACKEND_VLLM = vllm|python|.|false|true
 BACKEND_VLLM_OMNI = vllm-omni|python|.|false|true
+BACKEND_LONGCAT_VIDEO = longcat-video|python|.|--progress=plain|true
 BACKEND_SGLANG = sglang|python|.|false|true
 BACKEND_DIFFUSERS = diffusers|python|.|--progress=plain|true
 BACKEND_CHATTERBOX = chatterbox|python|.|false|true
@@ -1339,6 +1342,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_NEUTTS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_KOKORO)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VLLM)))
 $(eval $(call generate-docker-build-target,$(BACKEND_VLLM_OMNI)))
+$(eval $(call generate-docker-build-target,$(BACKEND_LONGCAT_VIDEO)))
 $(eval $(call generate-docker-build-target,$(BACKEND_SGLANG)))
 $(eval $(call generate-docker-build-target,$(BACKEND_DIFFUSERS)))
 $(eval $(call generate-docker-build-target,$(BACKEND_CHATTERBOX)))
@@ -1375,7 +1379,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC)))
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
-docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter
+docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-privacy-filter
 
 ########################################################
 ### Mock Backend for E2E Tests

README.md

@@ -235,7 +235,7 @@ Most backends wrap a best-in-class upstream engine. A handful of them are native
 | [moss-transcribe.cpp](https://github.com/mudler/moss-transcribe.cpp) | C++/GGML port of OpenMOSS MOSS-Transcribe-Diarize: joint long-form transcription, speaker diarization and timestamping in a single pass |
 | [ced.cpp](https://github.com/mudler/ced.cpp) | C++/GGML port of the CED audio-tagging models: sound-event classification (527-class AudioSet) over REST and the realtime API for live recognition |
 | [voice-detect.cpp](https://github.com/mudler/voice-detect.cpp) | Speaker recognition and voice analysis (ECAPA-TDNN, WeSpeaker, ERes2Net, CAM++, wav2vec2 age/gender/emotion), replacing the Python speaker-recognition backend |
-| [voxtral.c](https://github.com/mudler/voxtral.c) | Voxtral Realtime 4B speech-to-text in pure C |
+| [voxtral-tts.c](https://github.com/mudler/voxtral-tts.c) | Voxtral Realtime 4B speech-to-text in pure C |
 | [vibevoice.cpp](https://github.com/mudler/vibevoice.cpp) | Native port of Microsoft VibeVoice for TTS (voice cloning) and long-form ASR with speaker diarization |
 | [rf-detr.cpp](https://github.com/mudler/rf-detr.cpp) | Native RF-DETR object detection and instance segmentation |
 | [locate-anything.cpp](https://github.com/mudler/locate-anything.cpp) | Open-vocabulary object detection and visual grounding (LocateAnything-3B) |

backend/Dockerfile.llama-cpp

@@ -111,6 +111,10 @@ RUN make -BC /LocalAI/backend/cpp/llama-cpp package
 # ============================================================================
 FROM ${BUILDER_BASE_IMAGE} AS builder-prebuilt
 
+ARG APT_MIRROR
+ENV APT_MIRROR=${APT_MIRROR}
+ARG APT_PORTS_MIRROR
+ENV APT_PORTS_MIRROR=${APT_PORTS_MIRROR}
 ARG BUILD_TYPE
 ENV BUILD_TYPE=${BUILD_TYPE}
 ARG CUDA_DOCKER_ARCH

backend/README.md

@@ -46,6 +46,7 @@ The backend system provides language-specific Dockerfiles that handle the build
 - **vllm**: High-performance LLM inference
 - **mlx**: Apple Silicon optimization
 - **diffusers**: Stable Diffusion models
+- **longcat-video**: CUDA text/image-to-video and speech-driven avatar generation
 - **Audio**: coqui, faster-whisper, kitten-tts
 - **Vision**: mlx-vlm, rfdetr
 - **Specialized**: rerankers, chatterbox, kokoro

backend/backend.proto

@@ -448,6 +448,11 @@ message ModelOptions {
   // Proxy carries the cloud-proxy backend's per-model configuration.
   // Empty for non-proxy backends.
   ProxyOptions Proxy = 74;
+
+  // EnableScore reserves backend resources for the Score RPC. It is derived
+  // from the model's explicit `known_usecases: [score]` declaration so models
+  // that never score retain their ordinary serving footprint.
+  bool EnableScore = 75;
 }
 
 // ProxyOptions configures the cloud-proxy backend. UpstreamURL and
@@ -577,6 +582,10 @@ message GenerateVideoRequest {
   float cfg_scale = 10;    // Classifier-free guidance scale
   int32 step = 11;         // Number of inference steps
   string dst = 12;        // Output path for the generated video
+  string audio = 13;      // Path to staged audio for audio-conditioned video
+  // Backend-specific per-request generation parameters. Values are strings
+  // and are validated/coerced by the selected backend.
+  map<string, string> params = 14;
 }
 
 message TTSRequest {
@@ -1256,4 +1265,3 @@ message ForwardReply {
   repeated ForwardHeader headers = 2;
   bytes body_chunk = 3;
 }
-

backend/cpp/ds4/package.sh

@@ -1,12 +1,14 @@
 #!/bin/bash
-set -e
+set -euo pipefail
 CURDIR=$(dirname "$(realpath "$0")")
 REPO_ROOT="${CURDIR}/../../.."
+PACKAGE_DIR="$CURDIR/package"
 
-mkdir -p "$CURDIR/package/lib"
-cp -avf "$CURDIR/grpc-server" "$CURDIR/package/"
-cp -avf "$CURDIR/ds4-worker"  "$CURDIR/package/"
-cp -rfv "$CURDIR/run.sh"      "$CURDIR/package/"
+rm -rf "$PACKAGE_DIR"
+mkdir -p "$PACKAGE_DIR/lib"
+cp -avf "$CURDIR/grpc-server" "$PACKAGE_DIR/"
+cp -avf "$CURDIR/ds4-worker"  "$PACKAGE_DIR/"
+cp -rfv "$CURDIR/run.sh"      "$PACKAGE_DIR/"
 
 UNAME_S=$(uname -s)
 if [ "$UNAME_S" = "Darwin" ]; then
@@ -16,25 +18,54 @@ if [ "$UNAME_S" = "Darwin" ]; then
 fi
 
 if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
-    cp -arfLv /lib64/ld-linux-x86-64.so.2 "$CURDIR/package/lib/ld.so"
-    LIBDIR=/lib/x86_64-linux-gnu
+    cp -arfLv /lib64/ld-linux-x86-64.so.2 "$PACKAGE_DIR/lib/ld.so"
 elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
-    cp -arfLv /lib/ld-linux-aarch64.so.1 "$CURDIR/package/lib/ld.so"
-    LIBDIR=/lib/aarch64-linux-gnu
+    cp -arfLv /lib/ld-linux-aarch64.so.1 "$PACKAGE_DIR/lib/ld.so"
 else
     echo "package.sh: unknown architecture" >&2; exit 1
 fi
 
-for lib in libc.so.6 libgcc_s.so.1 libstdc++.so.6 libm.so.6 libgomp.so.1 \
-           libdl.so.2 librt.so.1 libpthread.so.0; do
-    cp -arfLv "$LIBDIR/$lib" "$CURDIR/package/lib/$lib"
+# Bundle the complete dependency closure for both executables. In particular,
+# grpc-server links the distro gRPC/protobuf/absl stack; copying only the core
+# C/C++ runtime libraries leaves the scratch image unable to start.
+{
+    ldd "$CURDIR/grpc-server"
+    ldd "$CURDIR/ds4-worker"
+} | awk '$2 == "=>" && $3 ~ /^\// { print $3 }' | sort -u | \
+while read -r so; do
+    cp -arfLv "$so" "$PACKAGE_DIR/lib/"
 done
 
 GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
 if [ -f "$GPU_LIB_SCRIPT" ]; then
-    source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
+    # shellcheck source=/dev/null
+    source "$GPU_LIB_SCRIPT" "$PACKAGE_DIR/lib"
     package_gpu_libs
 fi
 
+# Resolve every dependency through the same loader and library path used by
+# the from-scratch image. The loader can still search host defaults, so reject
+# any absolute dependency path that escapes the package instead of accepting a
+# false-positive validation against a library that scratch will not contain.
+validate_packaged_binary() {
+    local binary="$1"
+    local resolution
+    resolution=$("$PACKAGE_DIR/lib/ld.so" \
+        --library-path "$PACKAGE_DIR/lib" \
+        --list "$PACKAGE_DIR/$binary")
+
+    printf '%s\n' "$resolution" | awk -v prefix="$PACKAGE_DIR/lib/" '
+        $2 == "=>" && $3 ~ /^\// && index($3, prefix) != 1 {
+            print "package.sh: dependency resolved outside package: " $0 > "/dev/stderr"
+            invalid = 1
+        }
+        END { exit invalid }
+    '
+}
+
+for binary in grpc-server ds4-worker; do
+    validate_packaged_binary "$binary"
+done
+
 echo "ds4 package contents:"
-ls -lah "$CURDIR/package/" "$CURDIR/package/lib/"
+ls -lah "$PACKAGE_DIR/" "$PACKAGE_DIR/lib/"

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=6198a356a85ed71534c02a9c1026203389f341e5
+IK_LLAMA_VERSION?=1fddd12ba861c4815a8633f14d9c5670692099cc
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=4f37f519722aa3242eecb7649466b4a4a2d6d6da
+LLAMA_VERSION?=505b1ed15ca80e2a19f12ff4ac365e40fb374053
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=