From dca0b6d269b8582111cacd80fa5f55ca08b62c7d Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@localai.io>
Date: Wed, 29 Jul 2026 18:15:33 +0000
Subject: [PATCH 1/7] feat(backends): add CachyLLaMA support

Add CachyLLaMA as a GGUF-compatible llama.cpp fork backend with CPU and Vulkan builds on Linux and Metal on Apple silicon. Expose it through model import, document persistent SSD cache options, and wire CI path filtering and dependency updates.

Assisted-by: Codex:gpt-5
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
---
 .docker/cachyllama-compile.sh                 |  39 +++
 .github/backend-matrix.yml                    |  65 +++++
 .github/workflows/backend_build_darwin.yml    |   8 +-
 .github/workflows/bump_deps.yaml              |   4 +
 Makefile                                      |  11 +-
 backend/Dockerfile.cachyllama                 | 160 ++++++++++++
 backend/cpp/cachyllama/Makefile               |  99 ++++++++
 backend/cpp/cachyllama/package.sh             |  66 +++++
 backend/cpp/cachyllama/run.sh                 |  56 +++++
 backend/index.yaml                            | 178 +++++++++++++
 core/gallery/importers/importers_test.go      |   2 +-
 core/gallery/importers/llama-cpp.go           |   3 +-
 core/gallery/importers/llama-cpp_test.go      |  21 +-
 .../e2e/import-form-ux-batch-d.spec.js        |   9 +
 core/http/react-ui/src/pages/ImportModel.jsx  |   4 +-
 docs/content/advanced/model-configuration.md  |   2 +-
 docs/content/features/text-generation.md      |  47 ++++
 docs/content/reference/compatibility-table.md |   1 +
 gallery/backends/index.yaml                   | 236 ++++++++++++++++++
 scripts/build/cachyllama-darwin.sh            |  82 ++++++
 scripts/lib/backend-filter.mjs                |  22 +-
 scripts/lib/backend-filter_test.mjs           |  32 ++-
 22 files changed, 1134 insertions(+), 13 deletions(-)
 create mode 100755 .docker/cachyllama-compile.sh
 create mode 100644 backend/Dockerfile.cachyllama
 create mode 100644 backend/cpp/cachyllama/Makefile
 create mode 100755 backend/cpp/cachyllama/package.sh
 create mode 100755 backend/cpp/cachyllama/run.sh
 create mode 100755 scripts/build/cachyllama-darwin.sh

diff --git a/.docker/cachyllama-compile.sh b/.docker/cachyllama-compile.sh
new file mode 100755
index 000000000000..3d4c9d2e728f
--- /dev/null
+++ b/.docker/cachyllama-compile.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Shared compile logic for backend/Dockerfile.cachyllama.
+# Sourced (via bind mount) from both builder-fromsource and builder-prebuilt stages.
+
+set -euxo pipefail
+
+export CCACHE_DIR=/root/.ccache
+ccache --max-size=5G || true
+ccache -z || true
+
+export CMAKE_ARGS="${CMAKE_ARGS:-} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache"
+
+if [[ -n "${CUDA_DOCKER_ARCH:-}" ]]; then
+  CUDA_ARCH_ESC="${CUDA_DOCKER_ARCH//;/\\;}"
+  export CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH_ESC}"
+  echo "CMAKE_ARGS(env) = ${CMAKE_ARGS}"
+  rm -rf /LocalAI/backend/cpp/cachyllama-*-build
+fi
+
+cd /LocalAI/backend/cpp/cachyllama
+
+if [ -z "${BUILD_TYPE:-}" ]; then
+  # Pure CPU image: one ggml CPU_ALL_VARIANTS build replaces the per-microarch binaries.
+  # arm64: the armv9.2 SME variants need gcc-14 (gcc-13 rejects +sme).
+  if [ "${TARGETARCH}" = "arm64" ]; then
+    apt-get update -qq && apt-get install -y -qq gcc-14 g++-14
+    export CC=gcc-14 CXX=g++-14
+  fi
+  make cachyllama-cpu-all
+else
+  # GPU build (cublas/hipblas/sycl/vulkan/...): single fallback CPU build, the accelerator
+  # does the compute. Keeps the GPU compile from also building the CPU variant matrix and
+  # avoids the gcc-14 apt step on GPU base images such as nvidia l4t.
+  make cachyllama-fallback
+fi
+make cachyllama-grpc
+make cachyllama-rpc-server
+
+ccache -s || true
diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml
index 293bc6978444..2101657d2112 100644
--- a/.github/backend-matrix.yml
+++ b/.github/backend-matrix.yml
@@ -6631,6 +6631,68 @@ include:
     context: "./"
     ubuntu-version: '2404'
 
+  # CachyLLaMA targets the lower-spec/APU use case with CPU and Vulkan builds.
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-cachyllama'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-amd64'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cachyllama"
+    dockerfile: "./backend/Dockerfile.cachyllama"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: ''
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-cpu-cachyllama'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-arm64'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cachyllama"
+    dockerfile: "./backend/Dockerfile.cachyllama"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    platform-tag: 'amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-cachyllama'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-vulkan-amd64'
+    runs-on: 'ubuntu-latest'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cachyllama"
+    dockerfile: "./backend/Dockerfile.cachyllama"
+    context: "./"
+    ubuntu-version: '2404'
+  - build-type: 'vulkan'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/arm64'
+    platform-tag: 'arm64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-vulkan-cachyllama'
+    builder-base-image: 'quay.io/go-skynet/ci-cache:base-grpc-vulkan-arm64'
+    runs-on: 'ubuntu-24.04-arm'
+    base-image: "ubuntu:24.04"
+    skip-drivers: 'false'
+    backend: "cachyllama"
+    dockerfile: "./backend/Dockerfile.cachyllama"
+    context: "./"
+    ubuntu-version: '2404'
+
 # Darwin matrix (consumed by backend-jobs-darwin).
 includeDarwin:
   - backend: "kimodocpp"
@@ -6668,6 +6730,9 @@ includeDarwin:
   - backend: "llama-cpp"
     tag-suffix: "-metal-darwin-arm64-llama-cpp"
     lang: "go"
+  - backend: "cachyllama"
+    tag-suffix: "-metal-darwin-arm64-cachyllama"
+    lang: "go"
   - backend: "stablediffusion-ggml"
     tag-suffix: "-metal-darwin-arm64-stablediffusion-ggml"
     build-type: "metal"
diff --git a/.github/workflows/backend_build_darwin.yml b/.github/workflows/backend_build_darwin.yml
index 2fcf097b00ff..1322e0daeefa 100644
--- a/.github/workflows/backend_build_darwin.yml
+++ b/.github/workflows/backend_build_darwin.yml
@@ -231,6 +231,12 @@ jobs:
           make protogen-go
           make backends/llama-cpp-darwin
 
+      - name: Build CachyLLaMA backend (Darwin Metal)
+        if: inputs.backend == 'cachyllama'
+        run: |
+          make protogen-go
+          make backends/cachyllama-darwin
+
       - name: Build ds4 backend (Darwin Metal)
         if: inputs.backend == 'ds4'
         run: |
@@ -257,7 +263,7 @@ jobs:
           make backends/audio-cpp-darwin
 
       - name: Build ${{ inputs.backend }}-darwin
-        if: inputs.backend != 'llama-cpp' && inputs.backend != 'ds4' && inputs.backend != 'privacy-filter' && inputs.backend != 'audio-cpp'
+        if: inputs.backend != 'llama-cpp' && inputs.backend != 'cachyllama' && inputs.backend != 'ds4' && inputs.backend != 'privacy-filter' && inputs.backend != 'audio-cpp'
         run: |
           make protogen-go
           BACKEND=${{ inputs.backend }} BUILD_TYPE=${{ inputs.build-type }} USE_PIP=${{ inputs.use-pip }} make build-darwin-${{ inputs.lang }}-backend
diff --git a/.github/workflows/bump_deps.yaml b/.github/workflows/bump_deps.yaml
index de3a8b74d515..dfc3df7751ca 100644
--- a/.github/workflows/bump_deps.yaml
+++ b/.github/workflows/bump_deps.yaml
@@ -22,6 +22,10 @@ jobs:
             variable: "TURBOQUANT_VERSION"
             branch: "feature/turboquant-kv-cache"
             file: "backend/cpp/turboquant/Makefile"
+          - repository: "fewtarius/CachyLLama"
+            variable: "CACHYLLAMA_VERSION"
+            branch: "master"
+            file: "backend/cpp/cachyllama/Makefile"
           - repository: "PrismML-Eng/llama.cpp"
             variable: "BONSAI_VERSION"
             branch: "prism"
diff --git a/Makefile b/Makefile
index 8739e1dac11a..fd2f22a60f5b 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 # Disable parallel execution for backend builds
-.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/qdrant-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
+.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/cachyllama backends/cachyllama-darwin backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/qdrant-store backends/cloud-proxy backends/localai-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin
 .NOTPARALLEL: backends/whisper-medusa
 .NOTPARALLEL: backends/funasr
 
@@ -1270,6 +1270,10 @@ backends/llama-cpp-darwin: build
 	bash ./scripts/build/llama-cpp-darwin.sh
 	./local-ai backends install "ocifile://$(abspath ./backend-images/llama-cpp.tar)"
 
+backends/cachyllama-darwin: build
+	bash ./scripts/build/cachyllama-darwin.sh
+	./local-ai backends install "ocifile://$(abspath ./backend-images/cachyllama.tar)"
+
 backends/ds4-darwin: build
 	bash ./scripts/build/ds4-darwin.sh
 	./local-ai backends install "ocifile://$(abspath ./backend-images/ds4.tar)"
@@ -1331,6 +1335,8 @@ BACKEND_IK_LLAMA_CPP = ik-llama-cpp|ik-llama-cpp|.|false|false
 # turboquant is a llama.cpp fork with TurboQuant KV-cache quantization.
 # Reuses backend/cpp/llama-cpp grpc-server sources via a thin wrapper Makefile.
 BACKEND_TURBOQUANT = turboquant|turboquant|.|false|false
+# CachyLLaMA is a llama.cpp fork with persistent SSD prompt caching and APU tuning.
+BACKEND_CACHYLLAMA = cachyllama|cachyllama|.|false|false
 # bonsai is a llama.cpp fork (PrismML) adding the Q1_0 (1-bit) and Q2_0 (ternary)
 # weight-quant kernels the Bonsai / Ternary-Bonsai models ship in. Reuses
 # backend/cpp/llama-cpp grpc-server sources via a thin wrapper Makefile.
@@ -1455,6 +1461,7 @@ endef
 $(eval $(call generate-docker-build-target,$(BACKEND_LLAMA_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_IK_LLAMA_CPP)))
 $(eval $(call generate-docker-build-target,$(BACKEND_TURBOQUANT)))
+$(eval $(call generate-docker-build-target,$(BACKEND_CACHYLLAMA)))
 $(eval $(call generate-docker-build-target,$(BACKEND_BONSAI)))
 $(eval $(call generate-docker-build-target,$(BACKEND_DS4)))
 $(eval $(call generate-docker-build-target,$(BACKEND_PRIVACY_FILTER)))
@@ -1548,7 +1555,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC)))
 docker-save-%: backend-images
 	docker save local-ai-backend:$* -o backend-images/$*.tar
 
-docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-localai-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-nemo-speech-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store docker-build-qdrant-store docker-build-audio-cpp
+docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-cachyllama docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-localai-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-nemo-speech-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store docker-build-qdrant-store docker-build-audio-cpp
 docker-build-backends: docker-build-whisper-medusa
 docker-build-backends: docker-build-funasr
 
diff --git a/backend/Dockerfile.cachyllama b/backend/Dockerfile.cachyllama
new file mode 100644
index 000000000000..ab2126185638
--- /dev/null
+++ b/backend/Dockerfile.cachyllama
@@ -0,0 +1,160 @@
+ARG BASE_IMAGE=ubuntu:24.04
+# BUILDER_BASE_IMAGE defaults to BASE_IMAGE so the Dockerfile parses even
+# when no prebuilt base is supplied. The builder-prebuilt stage is only
+# entered when BUILDER_TARGET=builder-prebuilt, so a "wrong" fallback
+# content here is harmless — BuildKit prunes the unreferenced builder.
+ARG BUILDER_BASE_IMAGE=${BASE_IMAGE}
+# BUILDER_TARGET selects which builder stage the final scratch image copies
+# package output from. Declared at global scope (before any FROM) so it's
+# usable in `FROM ${BUILDER_TARGET}` below. Default keeps local
+# `make backends/cachyllama` on the from-source path.
+ARG BUILDER_TARGET=builder-fromsource
+ARG APT_MIRROR=""
+ARG APT_PORTS_MIRROR=""
+
+
+# ============================================================================
+# Stage: builder-fromsource — self-contained build path.
+# Runs .docker/install-base-deps.sh (apt deps + cmake + protoc + gRPC +
+# conditional CUDA/ROCm/Vulkan), copies /opt/grpc to /usr/local, then
+# compiles the variant. Used when BUILDER_TARGET=builder-fromsource (the
+# default; local `make backends/cachyllama`).
+#
+# The install script is the same one that backend/Dockerfile.base-grpc-builder
+# runs, so the result is bit-equivalent to the prebuilt-base path
+# (builder-prebuilt below).
+# ============================================================================
+FROM ${BASE_IMAGE} AS builder-fromsource
+ARG BUILD_TYPE
+ARG CUDA_MAJOR_VERSION
+ARG CUDA_MINOR_VERSION
+ARG CMAKE_FROM_SOURCE=false
+# CUDA Toolkit 13.x compatibility: CMake 3.31.9+ fixes toolchain detection/arch table issues
+ARG CMAKE_VERSION=3.31.10
+ARG GRPC_VERSION=v1.65.0
+ARG GRPC_MAKEFLAGS="-j4 -Otarget"
+ARG SKIP_DRIVERS=false
+ARG TARGETARCH
+ARG TARGETVARIANT
+ARG GO_VERSION=1.25.4
+ARG UBUNTU_VERSION=2404
+ARG APT_MIRROR
+ARG APT_PORTS_MIRROR
+ARG AMDGPU_TARGETS=""
+ARG BACKEND=rerankers
+# CUDA target archs, e.g. --build-arg CUDA_DOCKER_ARCH='75;86;89;120'
+ARG CUDA_DOCKER_ARCH
+ARG CMAKE_ARGS
+
+ENV BUILD_TYPE=${BUILD_TYPE} \
+    CUDA_MAJOR_VERSION=${CUDA_MAJOR_VERSION} \
+    CUDA_MINOR_VERSION=${CUDA_MINOR_VERSION} \
+    CMAKE_FROM_SOURCE=${CMAKE_FROM_SOURCE} \
+    CMAKE_VERSION=${CMAKE_VERSION} \
+    GRPC_VERSION=${GRPC_VERSION} \
+    GRPC_MAKEFLAGS=${GRPC_MAKEFLAGS} \
+    SKIP_DRIVERS=${SKIP_DRIVERS} \
+    TARGETARCH=${TARGETARCH} \
+    UBUNTU_VERSION=${UBUNTU_VERSION} \
+    APT_MIRROR=${APT_MIRROR} \
+    APT_PORTS_MIRROR=${APT_PORTS_MIRROR} \
+    AMDGPU_TARGETS=${AMDGPU_TARGETS} \
+    CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH} \
+    CMAKE_ARGS=${CMAKE_ARGS} \
+    DEBIAN_FRONTEND=noninteractive
+
+# CUDA on PATH (no-op when CUDA isn't installed)
+ENV PATH=/usr/local/cuda/bin:${PATH}
+# HipBLAS / ROCm on PATH (no-op when ROCm isn't installed)
+ENV PATH=/opt/rocm/bin:${PATH}
+
+WORKDIR /build
+
+# Install everything via the shared script — the same one that
+# backend/Dockerfile.base-grpc-builder runs, so the prebuilt CI base and
+# this from-source path are bit-equivalent.
+RUN --mount=type=bind,source=.docker/install-base-deps.sh,target=/usr/local/sbin/install-base-deps \
+    --mount=type=bind,source=.docker/apt-mirror.sh,target=/usr/local/sbin/apt-mirror \
+    bash /usr/local/sbin/install-base-deps
+
+# Mirror builder-prebuilt: copy gRPC from /opt/grpc to /usr/local so
+# CMake's find_package finds it at the canonical prefix the Makefile expects.
+RUN cp -a /opt/grpc/. /usr/local/
+
+COPY . /LocalAI
+
+# BuildKit cache mount for ccache. See Dockerfile.llama-cpp (commit 9228e5b4)
+# for rationale. cachyllama is a llama.cpp fork that reuses
+# backend/cpp/llama-cpp source via a thin wrapper Makefile, so MOST TUs
+# are content-identical to the upstream llama-cpp build. Sharing a cache
+# id with llama-cpp could give cross-fork hits — but for now keep them
+# separate so a regression in one doesn't poison the other. Revisit
+# sharing after measuring the actual hit rate.
+#
+# The compile body is shared with builder-prebuilt via .docker/cachyllama-compile.sh.
+RUN --mount=type=bind,source=.docker/cachyllama-compile.sh,target=/usr/local/sbin/compile.sh \
+    --mount=type=cache,target=/root/.ccache,id=cachyllama-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \
+    bash /usr/local/sbin/compile.sh
+
+
+# Copy libraries using a script to handle architecture differences
+RUN make -BC /LocalAI/backend/cpp/cachyllama package
+
+
+# ============================================================================
+# Stage: builder-prebuilt — uses the pre-built base from
+# quay.io/go-skynet/ci-cache:base-grpc-* (built by .github/workflows/base-images.yml).
+# That image already has gRPC at /opt/grpc + apt deps + CUDA/ROCm/Vulkan
+# pre-installed, so we just copy gRPC to /usr/local and compile. Used when
+# BUILDER_TARGET=builder-prebuilt (CI when the matrix entry sets
+# builder-base-image).
+# ============================================================================
+FROM ${BUILDER_BASE_IMAGE} AS builder-prebuilt
+
+ARG BUILD_TYPE
+ENV BUILD_TYPE=${BUILD_TYPE}
+ARG CUDA_DOCKER_ARCH
+ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
+ARG CMAKE_ARGS
+ENV CMAKE_ARGS=${CMAKE_ARGS}
+# AMDGPU_TARGETS must be forwarded into the env here too — backend/cpp/llama-cpp/Makefile
+# (which the cachyllama Makefile reuses via a sibling build dir) errors out when the var
+# is empty on a hipblas build, and the prebuilt path is what CI exercises most of the
+# time. The builder-fromsource stage above already does this; mirror it here.
+ARG AMDGPU_TARGETS
+ENV AMDGPU_TARGETS=${AMDGPU_TARGETS}
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+# The base-grpc-* image installs gRPC to /opt/grpc but doesn't copy it to
+# /usr/local. Mirror what the from-source path does so the compile step
+# can find gRPC at the canonical prefix the Makefile expects.
+RUN cp -a /opt/grpc/. /usr/local/
+
+COPY . /LocalAI
+
+RUN --mount=type=bind,source=.docker/cachyllama-compile.sh,target=/usr/local/sbin/compile.sh \
+    --mount=type=cache,target=/root/.ccache,id=cachyllama-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \
+    bash /usr/local/sbin/compile.sh
+
+RUN make -BC /LocalAI/backend/cpp/cachyllama package
+
+
+# ============================================================================
+# Final stage — copies package output from one of the two builders.
+# BUILDER_TARGET selects which one. BuildKit prunes the unreferenced builder.
+#
+# BuildKit doesn't support variable expansion in `COPY --from=` directly,
+# so we resolve the ARG by aliasing the chosen builder to a fixed stage
+# name via `FROM ${BUILDER_TARGET} AS builder` and then COPY --from=builder.
+# BUILDER_TARGET itself is declared as a global ARG at the top of this
+# file (required for use in FROM), so we just re-import it into this
+# stage's scope before the FROM directive.
+# ============================================================================
+FROM ${BUILDER_TARGET} AS builder
+
+FROM scratch
+
+
+# Copy all available binaries (the build process only creates the appropriate ones for the target architecture)
+COPY --from=builder /LocalAI/backend/cpp/cachyllama/package/. ./
diff --git a/backend/cpp/cachyllama/Makefile b/backend/cpp/cachyllama/Makefile
new file mode 100644
index 000000000000..6f71dbc5f64e
--- /dev/null
+++ b/backend/cpp/cachyllama/Makefile
@@ -0,0 +1,99 @@
+
+# Pinned to the HEAD of master on https://github.com/fewtarius/CachyLLama.
+# Auto-bumped nightly by .github/workflows/bump_deps.yaml.
+CACHYLLAMA_VERSION?=cea45f4222e1eea5c2c68388d7e75904a65b6778
+LLAMA_REPO?=https://github.com/fewtarius/CachyLLama
+
+CMAKE_ARGS?=
+BUILD_TYPE?=
+NATIVE?=false
+ONEAPI_VARS?=/opt/intel/oneapi/setvars.sh
+TARGET?=--target grpc-server
+JOBS?=$(shell nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
+ARCH?=$(shell uname -m)
+
+CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+LLAMA_CPP_DIR := $(CURRENT_MAKEFILE_DIR)/../llama-cpp
+
+GREEN := \033[0;32m
+RESET := \033[0m
+
+# cachyllama is a llama.cpp fork. Rather than duplicating grpc-server.cpp / CMakeLists.txt /
+# prepare.sh we reuse the ones in backend/cpp/llama-cpp, and only swap which repo+sha the
+# fetch step pulls. Each flavor target copies ../llama-cpp into a sibling ../cachyllama-<flavor>-build
+# directory, then invokes llama-cpp's own build-llama-cpp-grpc-server with LLAMA_REPO/LLAMA_VERSION
+# overridden to point at the fork.
+# Each flavor target:
+#   1. copies backend/cpp/llama-cpp/ (grpc-server.cpp + prepare.sh + CMakeLists.txt + Makefile)
+#      into a sibling cachyllama-<flavor>-build directory;
+#   2. clones the cachyllama fork into cachyllama-<flavor>-build/llama.cpp via the copy's
+#      own `llama.cpp` target, overriding LLAMA_REPO/LLAMA_VERSION;
+#   3. runs the copy's `grpc-server` target, which produces the binary we copy up as
+#      cachyllama-<flavor>.
+define cachyllama-build
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build
+	cp -rf $(LLAMA_CPP_DIR) $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build
+	# Drop patches vendored for upstream llama.cpp; CachyLLaMA tracks upstream
+	# closely but must not accidentally receive patches pinned to another SHA.
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build/patches
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build purge
+	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build/grpc-server.cpp
+	$(info $(GREEN)I cachyllama build info:$(1)$(RESET))
+	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(CACHYLLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build llama.cpp
+	CMAKE_ARGS="$(CMAKE_ARGS) $(2)" TARGET="$(3)" \
+	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(CACHYLLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build grpc-server
+	cp -rfv $(CURRENT_MAKEFILE_DIR)/../cachyllama-$(1)-build/grpc-server cachyllama-$(1)
+endef
+
+cachyllama-avx2:
+	$(call cachyllama-build,avx2,-DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on,--target grpc-server)
+
+cachyllama-avx512:
+	$(call cachyllama-build,avx512,-DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=on -DGGML_FMA=on -DGGML_F16C=on,--target grpc-server)
+
+cachyllama-avx:
+	$(call cachyllama-build,avx,-DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server)
+
+cachyllama-fallback:
+	$(call cachyllama-build,fallback,-DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server)
+
+# Single-build CPU backend via ggml CPU_ALL_VARIANTS (mirrors llama-cpp-cpu-all).
+# cachyllama reuses backend/cpp/llama-cpp's CMakeLists.txt (hw_grpc_proto STATIC) and
+# Makefile (SHARED_LIBS make-var + EXTRA_CMAKE_ARGS), so this passes the same overrides
+# through to the copied build: SHARED_LIBS=ON, the DL flags, and --target ggml (which
+# pulls in the per-microarch libggml-cpu-*.so via ggml's add_dependencies). The .so set
+# is collected for package.sh to bundle into package/lib.
+cachyllama-cpu-all:
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build
+	cp -rf $(LLAMA_CPP_DIR) $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build
+	# Drop patches vendored for upstream llama.cpp; they are SHA-specific.
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build/patches
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build purge
+	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build/grpc-server.cpp
+	$(info $(GREEN)I cachyllama build info:cpu-all-variants$(RESET))
+	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(CACHYLLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build llama.cpp
+	SHARED_LIBS=ON EXTRA_CMAKE_ARGS="-DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON" TARGET="--target grpc-server --target ggml" \
+	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(CACHYLLAMA_VERSION) \
+	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build grpc-server
+	cp -rfv $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build/grpc-server cachyllama-cpu-all
+	rm -rf ggml-shared-libs && mkdir -p ggml-shared-libs
+	find $(CURRENT_MAKEFILE_DIR)/../cachyllama-cpu-all-build/llama.cpp/build \( -name '*.so*' -o -name '*.dylib' \) -exec cp -av {} ggml-shared-libs/ \;
+	@echo "Collected ggml shared backends:" && ls -la ggml-shared-libs/
+
+cachyllama-grpc:
+	$(call cachyllama-build,grpc,-DGGML_RPC=ON -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server --target ggml-rpc-server)
+
+cachyllama-rpc-server: cachyllama-grpc
+	cp -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-grpc-build/llama.cpp/build/bin/ggml-rpc-server cachyllama-rpc-server
+
+package:
+	bash package.sh
+
+purge:
+	rm -rf $(CURRENT_MAKEFILE_DIR)/../cachyllama-*-build
+	rm -rf cachyllama-* package
+
+clean: purge
diff --git a/backend/cpp/cachyllama/package.sh b/backend/cpp/cachyllama/package.sh
new file mode 100755
index 000000000000..daedf6d910c3
--- /dev/null
+++ b/backend/cpp/cachyllama/package.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Script to copy the appropriate libraries based on architecture
+# This script is used in the final stage of the Dockerfile
+
+set -e
+
+CURDIR=$(dirname "$(realpath $0)")
+REPO_ROOT="${CURDIR}/../../.."
+
+# Create lib directory
+mkdir -p $CURDIR/package/lib
+
+cp -avrf $CURDIR/cachyllama-* $CURDIR/package/
+cp -rfv $CURDIR/run.sh $CURDIR/package/
+
+# Bundle the ggml shared backends from the CPU_ALL_VARIANTS build into package/lib. ggml
+# discovers the per-microarch libggml-cpu-*.so by scanning the executable directory, which
+# (via the bundled lib/ld.so that run.sh launches through) resolves to lib/. See the
+# matching comment in backend/cpp/llama-cpp/package.sh. No-op on the fallback/ROCm builds.
+if [ -d "$CURDIR/ggml-shared-libs" ]; then
+    echo "Bundling ggml shared backends (CPU_ALL_VARIANTS)..."
+    cp -avf $CURDIR/ggml-shared-libs/*.so* $CURDIR/package/lib/
+fi
+
+# Detect architecture and copy appropriate libraries
+if [ -f "/lib64/ld-linux-x86-64.so.2" ]; then
+    # x86_64 architecture
+    echo "Detected x86_64 architecture, copying x86_64 libraries..."
+    cp -arfLv /lib64/ld-linux-x86-64.so.2 $CURDIR/package/lib/ld.so
+    cp -arfLv /lib/x86_64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp -arfLv /lib/x86_64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
+    cp -arfLv /lib/x86_64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
+    cp -arfLv /lib/x86_64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
+elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then
+    # ARM64 architecture
+    echo "Detected ARM64 architecture, copying ARM64 libraries..."
+    cp -arfLv /lib/ld-linux-aarch64.so.1 $CURDIR/package/lib/ld.so
+    cp -arfLv /lib/aarch64-linux-gnu/libc.so.6 $CURDIR/package/lib/libc.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libgcc_s.so.1 $CURDIR/package/lib/libgcc_s.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libstdc++.so.6 $CURDIR/package/lib/libstdc++.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libm.so.6 $CURDIR/package/lib/libm.so.6
+    cp -arfLv /lib/aarch64-linux-gnu/libgomp.so.1 $CURDIR/package/lib/libgomp.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libdl.so.2 $CURDIR/package/lib/libdl.so.2
+    cp -arfLv /lib/aarch64-linux-gnu/librt.so.1 $CURDIR/package/lib/librt.so.1
+    cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0
+else
+    echo "Error: Could not detect architecture"
+    exit 1
+fi
+
+# Package GPU libraries based on BUILD_TYPE
+GPU_LIB_SCRIPT="${REPO_ROOT}/scripts/build/package-gpu-libs.sh"
+if [ -f "$GPU_LIB_SCRIPT" ]; then
+    echo "Packaging GPU libraries for BUILD_TYPE=${BUILD_TYPE:-cpu}..."
+    source "$GPU_LIB_SCRIPT" "$CURDIR/package/lib"
+    package_gpu_libs
+fi
+
+echo "Packaging completed successfully"
+ls -liah $CURDIR/package/
+ls -liah $CURDIR/package/lib/
diff --git a/backend/cpp/cachyllama/run.sh b/backend/cpp/cachyllama/run.sh
new file mode 100755
index 000000000000..4a4c7453a37c
--- /dev/null
+++ b/backend/cpp/cachyllama/run.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+set -ex
+
+# Get the absolute current dir where the script is located
+CURDIR=$(dirname "$(realpath "$0")")
+
+cd /
+
+echo "CPU info:"
+grep -e "model\sname" /proc/cpuinfo | head -1
+grep -e "flags" /proc/cpuinfo | head -1
+
+BINARY=cachyllama-fallback
+
+# x86/arm64 ship a single cachyllama-cpu-all built with ggml CPU_ALL_VARIANTS: ggml's
+# backend registry dlopens the best libggml-cpu-*.so for this host, so no shell-side
+# probing. ROCm ships only cachyllama-fallback, so fall back to it when cpu-all is absent.
+if [ -e "$CURDIR"/cachyllama-cpu-all ]; then
+	BINARY=cachyllama-cpu-all
+fi
+
+if [ -n "$LLAMACPP_GRPC_SERVERS" ]; then
+	if [ -e "$CURDIR"/cachyllama-grpc ]; then
+		BINARY=cachyllama-grpc
+	fi
+fi
+
+# Extend ld library path with the dir where this script is located/lib
+if [ "$(uname)" == "Darwin" ]; then
+	export DYLD_LIBRARY_PATH="$CURDIR"/lib:$DYLD_LIBRARY_PATH
+else
+	export LD_LIBRARY_PATH="$CURDIR"/lib:$LD_LIBRARY_PATH
+	# Tell rocBLAS where to find TensileLibrary data (GPU kernel tuning files)
+	if [ -d "$CURDIR/lib/rocblas/library" ]; then
+		export ROCBLAS_TENSILE_LIBPATH="$CURDIR"/lib/rocblas/library
+	fi
+	# Same for hipBLASLt (rocblaslt): the bundled libhipblaslt.so resolves its
+	# TensileLibrary_lazy_gfx*.dat kernel data relative to itself, so point it at
+	# the bundled data or it falls back to slow generic kernels (issue #10660).
+	if [ -d "$CURDIR/lib/hipblaslt/library" ]; then
+		export HIPBLASLT_TENSILE_LIBPATH="$CURDIR"/lib/hipblaslt/library
+	fi
+fi
+
+# If there is a lib/ld.so, use it
+if [ -f "$CURDIR"/lib/ld.so ]; then
+	echo "Using lib/ld.so"
+	echo "Using binary: $BINARY"
+	exec "$CURDIR"/lib/ld.so "$CURDIR"/$BINARY "$@"
+fi
+
+echo "Using binary: $BINARY"
+exec "$CURDIR"/$BINARY "$@"
+
+# We should never reach this point, however just in case we do, run fallback
+exec "$CURDIR"/cachyllama-fallback "$@"
diff --git a/backend/index.yaml b/backend/index.yaml
index 1bf606e65ea8..5e4e04663dfc 100644
--- a/backend/index.yaml
+++ b/backend/index.yaml
@@ -29389,3 +29389,181 @@
   uri: quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-funasr
   mirrors:
     - localai/localai-backends:master-metal-darwin-arm64-funasr
+- name: cachyllama
+  alias: cachyllama
+  license: mit
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+- name: cachyllama-development
+  alias: cachyllama
+  license: mit
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  capabilities:
+    default: cpu-cachyllama-development
+    metal: metal-cachyllama-development
+    vulkan: vulkan-cachyllama-development
+- name: cpu-cachyllama
+  alias: cachyllama
+  license: mit
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  uri: quay.io/go-skynet/local-ai-backends:latest-cpu-cachyllama
+  mirrors:
+    - localai/localai-backends:latest-cpu-cachyllama
+- name: cpu-cachyllama-development
+  alias: cachyllama
+  license: mit
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  uri: quay.io/go-skynet/local-ai-backends:master-cpu-cachyllama
+  mirrors:
+    - localai/localai-backends:master-cpu-cachyllama
+- name: vulkan-cachyllama
+  alias: cachyllama
+  license: mit
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  uri: quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-cachyllama
+  mirrors:
+    - localai/localai-backends:latest-gpu-vulkan-cachyllama
+- name: vulkan-cachyllama-development
+  alias: cachyllama
+  license: mit
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  uri: quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-cachyllama
+  mirrors:
+    - localai/localai-backends:master-gpu-vulkan-cachyllama
+- name: metal-cachyllama
+  alias: cachyllama
+  license: mit
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  uri: quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-cachyllama
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-cachyllama
+- name: metal-cachyllama-development
+  alias: cachyllama
+  license: mit
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  uri: quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-cachyllama
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-cachyllama
diff --git a/core/gallery/importers/importers_test.go b/core/gallery/importers/importers_test.go
index 9b4ce07238da..684a17d5c7f3 100644
--- a/core/gallery/importers/importers_test.go
+++ b/core/gallery/importers/importers_test.go
@@ -302,7 +302,7 @@ var _ = Describe("DiscoverModelConfig", func() {
 				names = append(names, e.Name)
 				modalities = append(modalities, e.Modality)
 			}
-			Expect(names).To(ContainElements("ik-llama-cpp", "turboquant"))
+			Expect(names).To(ContainElements("ik-llama-cpp", "turboquant", "cachyllama"))
 			for _, m := range modalities {
 				Expect(m).To(Equal("text"))
 			}
diff --git a/core/gallery/importers/llama-cpp.go b/core/gallery/importers/llama-cpp.go
index c803f8ae2a25..d51c08ac5d0d 100644
--- a/core/gallery/importers/llama-cpp.go
+++ b/core/gallery/importers/llama-cpp.go
@@ -37,6 +37,7 @@ func (i *LlamaCPPImporter) AdditionalBackends() []KnownBackendEntry {
 	return []KnownBackendEntry{
 		{Name: "ik-llama-cpp", Modality: "text", Description: "GGUF drop-in replacement for llama-cpp with ik-quants"},
 		{Name: "turboquant", Modality: "text", Description: "GGUF drop-in replacement for llama-cpp with TurboQuant optimizations"},
+		{Name: "cachyllama", Modality: "text", Description: "GGUF drop-in replacement for llama-cpp with persistent SSD prompt caching and APU tuning"},
 		{Name: "vllm-cpp", Modality: "text", Description: "vLLM-style continuous-batching engine (vllm.cpp) consuming GGUF, by the LocalAI team"},
 	}
 }
@@ -136,7 +137,7 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
 	backend := "llama-cpp"
 	if b, ok := preferencesMap["backend"].(string); ok {
 		switch b {
-		case "ik-llama-cpp", "turboquant", "vllm-cpp":
+		case "ik-llama-cpp", "turboquant", "cachyllama", "vllm-cpp":
 			backend = b
 		}
 	}
diff --git a/core/gallery/importers/llama-cpp_test.go b/core/gallery/importers/llama-cpp_test.go
index 139b29884bb7..73e3076cd4b1 100644
--- a/core/gallery/importers/llama-cpp_test.go
+++ b/core/gallery/importers/llama-cpp_test.go
@@ -181,6 +181,23 @@ var _ = Describe("LlamaCPPImporter", func() {
 			Expect(modelConfig.Files[0].Filename).To(Equal("my-model.gguf"))
 		})
 
+		It("swaps the emitted backend to cachyllama when preferred", func() {
+			preferences := json.RawMessage(`{"backend": "cachyllama"}`)
+			details := Details{
+				URI:         "https://example.com/my-model.gguf",
+				Preferences: preferences,
+			}
+
+			modelConfig, err := importer.Import(details)
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: cachyllama"), fmt.Sprintf("Model config: %+v", modelConfig))
+			Expect(modelConfig.ConfigFile).NotTo(ContainSubstring("backend: llama-cpp\n"), fmt.Sprintf("Model config: %+v", modelConfig))
+			Expect(modelConfig.ConfigFile).To(ContainSubstring("model: my-model.gguf"), fmt.Sprintf("Model config: %+v", modelConfig))
+			Expect(len(modelConfig.Files)).To(Equal(1))
+			Expect(modelConfig.Files[0].Filename).To(Equal("my-model.gguf"))
+		})
+
 		It("swaps the emitted backend to vllm-cpp when preferred, keeping engine-side templating", func() {
 			preferences := json.RawMessage(`{"backend": "vllm-cpp"}`)
 			details := Details{
@@ -551,7 +568,7 @@ var _ = Describe("LlamaCPPImporter", func() {
 	})
 
 	Context("AdditionalBackends", func() {
-		It("advertises ik-llama-cpp, turboquant and vllm-cpp as drop-in replacements", func() {
+		It("advertises the curated GGUF drop-in replacements", func() {
 			entries := importer.AdditionalBackends()
 
 			names := make([]string, 0, len(entries))
@@ -560,7 +577,7 @@ var _ = Describe("LlamaCPPImporter", func() {
 				names = append(names, e.Name)
 				byName[e.Name] = e
 			}
-			Expect(names).To(ConsistOf("ik-llama-cpp", "turboquant", "vllm-cpp"))
+			Expect(names).To(ConsistOf("ik-llama-cpp", "turboquant", "cachyllama", "vllm-cpp"))
 
 			for _, name := range names {
 				e := byName[name]
diff --git a/core/http/react-ui/e2e/import-form-ux-batch-d.spec.js b/core/http/react-ui/e2e/import-form-ux-batch-d.spec.js
index 5ac9b7af2e6d..cdc7baded1c9 100644
--- a/core/http/react-ui/e2e/import-form-ux-batch-d.spec.js
+++ b/core/http/react-ui/e2e/import-form-ux-batch-d.spec.js
@@ -14,6 +14,7 @@ const MOCK_BACKENDS = [
   { name: 'llama-cpp', modality: 'text', auto_detect: true, installed: true },
   { name: 'ik-llama-cpp', modality: 'text', auto_detect: true, installed: true },
   { name: 'turboquant', modality: 'text', auto_detect: true, installed: true },
+  { name: 'cachyllama', modality: 'text', auto_detect: true, installed: true },
   { name: 'stablediffusion-ggml', modality: 'image', auto_detect: true, installed: true },
   { name: 'transformers', modality: 'text', auto_detect: true, installed: true },
   { name: 'sentencetransformers', modality: 'embeddings', auto_detect: true, installed: true },
@@ -91,6 +92,14 @@ test.describe('Import form UX — Batch D (progressive disclosure)', () => {
     await expect(modelTypeInput(page)).toHaveCount(0)
   })
 
+  test('D1 — selecting cachyllama shows GGUF Quantizations + MMProj', async ({ page }) => {
+    await enterPowerPreferences(page)
+    await selectBackend(page, 'cachyllama')
+    await expect(quantizationsInput(page)).toBeVisible()
+    await expect(mmprojInput(page)).toBeVisible()
+    await expect(modelTypeInput(page)).toHaveCount(0)
+  })
+
   test('D1 — selecting transformers hides Quantizations + MMProj, shows Model Type', async ({ page }) => {
     await openOptions(page)
     await selectBackend(page, 'transformers')
diff --git a/core/http/react-ui/src/pages/ImportModel.jsx b/core/http/react-ui/src/pages/ImportModel.jsx
index 18b9548cff12..77a91dce47ad 100644
--- a/core/http/react-ui/src/pages/ImportModel.jsx
+++ b/core/http/react-ui/src/pages/ImportModel.jsx
@@ -267,11 +267,11 @@ export default function ImportModel() {
   // flipping backends back and forth doesn't lose input.
   const showQuantizations = useMemo(() => {
     if (!prefs.backend) return true
-    return ['llama-cpp', 'ik-llama-cpp', 'turboquant', 'stablediffusion-ggml'].includes(prefs.backend)
+    return ['llama-cpp', 'ik-llama-cpp', 'turboquant', 'cachyllama', 'stablediffusion-ggml'].includes(prefs.backend)
   }, [prefs.backend])
   const showMmprojQuantizations = useMemo(() => {
     if (!prefs.backend) return true
-    return ['llama-cpp', 'ik-llama-cpp', 'turboquant'].includes(prefs.backend)
+    return ['llama-cpp', 'ik-llama-cpp', 'turboquant', 'cachyllama'].includes(prefs.backend)
   }, [prefs.backend])
   const showModelType = useMemo(() => {
     if (!prefs.backend) return true
diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md
index bea83b6f5abc..dc7023ec479e 100644
--- a/docs/content/advanced/model-configuration.md
+++ b/docs/content/advanced/model-configuration.md
@@ -844,7 +844,7 @@ These options apply when using the `vllm` backend:
 | `disable_log_stats` | bool | Disable logging statistics |
 | `dtype` | string | Data type (e.g., `float16`, `bfloat16`) |
 | `flash_attention` | string | Flash attention configuration |
-| `cache_type_k` | string | Key cache quantization type. Maps to llama.cpp's `-ctk`. Accepted values for llama.cpp-family backends (`llama-cpp`, `ik-llama-cpp`, `turboquant`): `f16`, `f32`, `q8_0`, `q4_0`, `q4_1`, `q5_0`, `q5_1`. The `turboquant` backend additionally accepts `turbo2`, `turbo3`, `turbo4` - the fork's TurboQuant KV-cache schemes. `turbo3`/`turbo4` auto-enable flash_attention. |
+| `cache_type_k` | string | Key cache quantization type. Maps to llama.cpp's `-ctk`. Accepted values for llama.cpp-family backends (`llama-cpp`, `ik-llama-cpp`, `turboquant`, `cachyllama`): `f16`, `f32`, `q8_0`, `q4_0`, `q4_1`, `q5_0`, `q5_1`. The `turboquant` backend additionally accepts `turbo2`, `turbo3`, `turbo4` - the fork's TurboQuant KV-cache schemes. `turbo3`/`turbo4` auto-enable flash_attention. |
 | `cache_type_v` | string | Value cache quantization type. Maps to llama.cpp's `-ctv`. Same accepted values as `cache_type_k`. Note: any quantized V cache requires flash_attention to be enabled. |
 | `limit_mm_per_prompt` | object | Limit multimodal content per prompt: `{image: int, video: int, audio: int}` |
 
diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md
index b133f75e4d24..5be278b13df7 100644
--- a/docs/content/features/text-generation.md
+++ b/docs/content/features/text-generation.md
@@ -810,6 +810,53 @@ The `cache_type_k` / `cache_type_v` fields map to llama.cpp's `-ctk` / `-ctv` fl
 - [llama-cpp-turboquant](https://github.com/TheTom/llama-cpp-turboquant)
 - [Tracked branch: `feature/turboquant-kv-cache`](https://github.com/TheTom/llama-cpp-turboquant/tree/feature/turboquant-kv-cache)
 
+### CachyLLaMA (persistent prompt cache for lower-spec hardware)
+
+[CachyLLaMA](https://github.com/fewtarius/CachyLLaMA) is a llama.cpp fork aimed at
+APUs, integrated GPUs, handhelds, and other shared-memory or lower-throughput
+systems. Its main addition is an SSD-backed KV cache that restores stable prompt
+prefixes across requests and process restarts.
+
+Install the alias and select it in model YAML like any other GGUF backend:
+
+```bash
+local-ai backends install cachyllama
+```
+
+```yaml
+name: cached-agent-model
+backend: cachyllama
+parameters:
+  model: model.gguf
+options:
+  - --cache-ssd:/models/.cachyllama-cache
+  - --cache-ssd-checkpoints:64
+  - --cache-ssd-hot-window:16384
+  - --cache-ssd-warm-window:32768
+  - --cache-ssd-page-size:1024
+  - --cache-ssd-system-prompts:8
+  - --cache-ssd-system-max-days:30
+```
+
+The cache directory must be on persistent storage and writable by LocalAI.
+Inside a container, put it below a mounted models or data volume.
+
+| Option | Meaning |
+|--------|---------|
+| `--cache-ssd` | Directory for SSD-backed KV checkpoints; enables the feature. |
+| `--cache-ssd-checkpoints` | Maximum checkpoints kept per slot. |
+| `--cache-ssd-hot-window` | Token window always retained in the hot tier. |
+| `--cache-ssd-warm-window` | Token window retained in RAM when possible. |
+| `--cache-ssd-max-cold` | Maximum cold checkpoints; `0` means unlimited. |
+| `--cache-ssd-page-size` | Tokens per cache page: `512`, `1024`, or `2048`. |
+| `--cache-ssd-max-conversations` | Maximum conversation directories; `0` means unlimited. |
+| `--cache-ssd-system-prompts` | Number of cross-conversation system prompts to cache. |
+| `--cache-ssd-system-max-days` | Expire unused system-prompt entries after this many days. |
+
+LocalAI publishes CPU and Vulkan images for Linux on amd64 and arm64, plus a
+Metal image for Apple silicon. The Vulkan build is intended for AMD APUs and
+other supported integrated GPUs.
+
 
 ### vLLM
 
diff --git a/docs/content/reference/compatibility-table.md b/docs/content/reference/compatibility-table.md
index 127de9e9427b..f106da7736e0 100644
--- a/docs/content/reference/compatibility-table.md
+++ b/docs/content/reference/compatibility-table.md
@@ -23,6 +23,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({
 | [llama.cpp](https://github.com/ggerganov/llama.cpp) | LLM inference in C/C++. Supports LLaMA, Mamba, RWKV, Falcon, Starcoder, GPT-2, [and many others](https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#description) | GPT, Functions | yes | yes | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Metal, Jetson L4T |
 | [ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp) | Hard fork of llama.cpp optimized for CPU/hybrid CPU+GPU with IQK quants, custom quant mixes, and MLA for DeepSeek | GPT | yes | yes | CPU (AVX2+) |
 | [turboquant](https://github.com/TheTom/llama-cpp-turboquant) | llama.cpp fork adding the TurboQuant KV-cache quantization scheme | GPT | yes | yes | CPU, CUDA 12/13, ROCm, Intel SYCL, Vulkan, Jetson L4T |
+| [CachyLLaMA](https://github.com/fewtarius/CachyLLaMA) | llama.cpp fork with persistent SSD-backed prompt caching and APU tuning | GPT | yes | yes | CPU, Vulkan, Metal |
 | [ds4](https://github.com/antirez/ds4) | DeepSeek V4 Flash single-model inference engine, optimized for Metal and CUDA | GPT | no | yes | CPU, CUDA 12/13, Metal, Jetson L4T |
 | [vllm.cpp](https://github.com/mudler/vllm.cpp) | From-scratch C++20 port of vLLM by the LocalAI team: paged KV cache, continuous batching, prefix caching, safetensors + GGUF, engine-enforced structured output, no Python at inference | GPT, Functions | no | yes | CPU, CUDA 13 (Ampere through Blackwell), Vulkan, Metal, Jetson L4T (Orin, Thor, GB10) |
 | [vLLM](https://github.com/vllm-project/vllm) | Fast LLM serving with PagedAttention; GPTQ/AWQ/FP8 quantization | GPT, Functions, Multimodal | no | yes | CUDA 12/13, ROCm, Intel SYCL, Jetson L4T |
diff --git a/gallery/backends/index.yaml b/gallery/backends/index.yaml
index 383d88dd45b3..7d278cdf49af 100644
--- a/gallery/backends/index.yaml
+++ b/gallery/backends/index.yaml
@@ -43107,3 +43107,239 @@
   urls:
     - https://github.com/modelscope/FunASR
   version: 0.0.0-development
+- alias: cachyllama
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  channel: latest
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  id: cachyllama
+  kind: backend
+  license: mit
+  name: cachyllama
+  schemaVersion: 1
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  version: 0.0.0-latest
+- alias: cachyllama
+  capabilities:
+    default: cpu-cachyllama-development
+    metal: metal-cachyllama-development
+    vulkan: vulkan-cachyllama-development
+  channel: development
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  id: cachyllama-development
+  kind: backend
+  license: mit
+  name: cachyllama-development
+  schemaVersion: 1
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  version: 0.0.0-development
+- alias: cachyllama
+  arch: amd64
+  artifact: oci://quay.io/go-skynet/local-ai-backends:latest-cpu-cachyllama
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  channel: latest
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  id: cpu-cachyllama
+  kind: backend
+  license: mit
+  mirrors:
+    - oci://localai/localai-backends:latest-cpu-cachyllama
+  mutable: true
+  name: cpu-cachyllama
+  os: linux
+  schemaVersion: 1
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  version: 0.0.0-latest
+- alias: cachyllama
+  arch: amd64
+  artifact: oci://quay.io/go-skynet/local-ai-backends:master-cpu-cachyllama
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  channel: development
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  id: cpu-cachyllama-development
+  kind: backend
+  license: mit
+  mirrors:
+    - oci://localai/localai-backends:master-cpu-cachyllama
+  mutable: true
+  name: cpu-cachyllama-development
+  os: linux
+  schemaVersion: 1
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  version: 0.0.0-development
+- alias: cachyllama
+  arch: amd64
+  artifact: oci://quay.io/go-skynet/local-ai-backends:latest-gpu-vulkan-cachyllama
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  channel: latest
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  id: vulkan-cachyllama
+  kind: backend
+  license: mit
+  mirrors:
+    - oci://localai/localai-backends:latest-gpu-vulkan-cachyllama
+  mutable: true
+  name: vulkan-cachyllama
+  os: linux
+  schemaVersion: 1
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  version: 0.0.0-latest
+- alias: cachyllama
+  arch: amd64
+  artifact: oci://quay.io/go-skynet/local-ai-backends:master-gpu-vulkan-cachyllama
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  channel: development
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  id: vulkan-cachyllama-development
+  kind: backend
+  license: mit
+  mirrors:
+    - oci://localai/localai-backends:master-gpu-vulkan-cachyllama
+  mutable: true
+  name: vulkan-cachyllama-development
+  os: linux
+  schemaVersion: 1
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  version: 0.0.0-development
+- alias: cachyllama
+  arch: arm64
+  artifact: oci://quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-cachyllama
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  channel: latest
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  id: metal-cachyllama
+  kind: backend
+  license: mit
+  mirrors:
+    - oci://localai/localai-backends:latest-metal-darwin-arm64-cachyllama
+  mutable: true
+  name: metal-cachyllama
+  os: darwin
+  schemaVersion: 1
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  version: 0.0.0-latest
+- alias: cachyllama
+  arch: arm64
+  artifact: oci://quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-cachyllama
+  capabilities:
+    default: cpu-cachyllama
+    metal: metal-cachyllama
+    vulkan: vulkan-cachyllama
+  channel: development
+  description: |
+    llama.cpp fork for lower-spec and shared-memory systems, with persistent
+    SSD-backed prompt caching and Vulkan APU tuning.
+  id: metal-cachyllama-development
+  kind: backend
+  license: mit
+  mirrors:
+    - oci://localai/localai-backends:master-metal-darwin-arm64-cachyllama
+  mutable: true
+  name: metal-cachyllama-development
+  os: darwin
+  schemaVersion: 1
+  tags:
+    - text-to-text
+    - LLM
+    - CPU
+    - GPU
+    - Vulkan
+    - Metal
+    - kv-cache
+  urls:
+    - https://github.com/fewtarius/CachyLLama
+  version: 0.0.0-development
diff --git a/scripts/build/cachyllama-darwin.sh b/scripts/build/cachyllama-darwin.sh
new file mode 100755
index 000000000000..3e024607220c
--- /dev/null
+++ b/scripts/build/cachyllama-darwin.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+
+set -ex
+
+IMAGE_NAME="${IMAGE_NAME:-localai/cachyllama-darwin}"
+
+pushd backend/cpp/cachyllama
+
+# Single build via ggml CPU_ALL_VARIANTS: one binary plus the per-microarch Apple/arm
+# dylibs (apple_m1/m2_m3/m4, armv8.x) that ggml selects at runtime. GGML_METAL stays ON
+# and --target ggml also builds ggml-metal (via add_dependencies), so the Metal GPU
+# backend is still produced as a loadable libggml-metal.dylib.
+make cachyllama-cpu-all && \
+make cachyllama-grpc && \
+make cachyllama-rpc-server
+
+popd
+
+mkdir -p build/darwin
+mkdir -p backend-images
+mkdir -p build/darwin/lib
+
+cp -rf backend/cpp/cachyllama/cachyllama-cpu-all build/darwin/
+cp -rf backend/cpp/cachyllama/cachyllama-grpc build/darwin/
+cp -rf backend/cpp/cachyllama/cachyllama-rpc-server build/darwin/
+
+# Distribute the shared ggml/llama libraries from the CPU_ALL_VARIANTS build. Unlike the
+# old fully-static fallback build, these have @rpath install names, so the otool loop below
+# (which only copies deps that exist on disk) will not pick them up. The split is by suffix:
+#  - ggml emits its loadable backends (per-microarch CPU variants, metal, blas) with a .so
+#    suffix EVEN ON DARWIN. These go in the package ROOT next to the binary, because darwin
+#    run.sh execs the binary directly (no bundled ld.so) so ggml's executable-directory
+#    scan looks there.
+#  - the core libraries (libggml-base/libggml/libllama/libllama-common/libmtmd) use the
+#    platform .dylib suffix and are NEEDED deps; they go in lib/, resolved at load time via
+#    the DYLD_LIBRARY_PATH=lib that run.sh exports. -a preserves the version symlinks.
+SHLIBS=backend/cpp/cachyllama/ggml-shared-libs
+cp -a $SHLIBS/*.so build/darwin/
+cp -a $SHLIBS/*.dylib build/darwin/lib/
+
+# Set default additional libs only for Darwin on M chips (arm64)
+if [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; then
+    ADDITIONAL_LIBS=${ADDITIONAL_LIBS:-$(ls /opt/homebrew/Cellar/protobuf/**/lib/libutf8_validity*.dylib 2>/dev/null)}
+else
+    ADDITIONAL_LIBS=${ADDITIONAL_LIBS:-""}
+fi
+
+for file in $ADDITIONAL_LIBS; do
+  cp -rfv $file build/darwin/lib
+done
+
+for file in build/darwin/*; do
+  LIBS="$(otool -L $file | awk 'NR > 1 { system("echo " $1) } ' | xargs echo)"
+  for lib in $LIBS; do
+    # only libraries ending in dylib
+    if [[ "$lib" == *.dylib ]]; then
+        if [ -e "$lib" ]; then
+            cp -rvf "$lib" build/darwin/lib
+        fi
+    fi
+  done
+done
+
+echo "--------------------------------"
+echo "ADDITIONAL_LIBS: $ADDITIONAL_LIBS"
+echo "--------------------------------"
+
+echo "Bundled libraries:"
+ls -la build/darwin/lib
+
+
+cp -rf backend/cpp/cachyllama/run.sh build/darwin/
+
+PLATFORMARCH="${PLATFORMARCH:-darwin/arm64}"
+
+./local-ai util create-oci-image \
+        build/darwin/. \
+        --output ./backend-images/cachyllama.tar \
+        --image-name $IMAGE_NAME \
+        --platform $PLATFORMARCH
+
+rm -rf build/darwin
diff --git a/scripts/lib/backend-filter.mjs b/scripts/lib/backend-filter.mjs
index 069433d3087a..5999e0f740de 100644
--- a/scripts/lib/backend-filter.mjs
+++ b/scripts/lib/backend-filter.mjs
@@ -78,6 +78,10 @@ export function inferBackendPath(item) {
     // via a thin wrapper Makefile. Changes to either dir should retrigger it.
     return `backend/cpp/turboquant/`;
   }
+  if (item.dockerfile.endsWith("cachyllama")) {
+    // CachyLLaMA is a llama.cpp fork that reuses the LocalAI gRPC sources.
+    return `backend/cpp/cachyllama/`;
+  }
   if (item.dockerfile.endsWith("bonsai")) {
     // bonsai is a llama.cpp fork that reuses backend/cpp/llama-cpp sources
     // via a thin wrapper Makefile. Changes to either dir should retrigger it.
@@ -105,6 +109,9 @@ export function inferBackendPathDarwin(item) {
   if (item.backend === "llama-cpp") {
     return `backend/cpp/llama-cpp/`;
   }
+  if (item.backend === "cachyllama") {
+    return `backend/cpp/cachyllama/`;
+  }
   // ds4 is C++ too (built via `make backends/ds4-darwin`); the matrix entry
   // carries lang=go for runner/toolchain selection, but the source is C++.
   if (item.backend === "ds4") {
@@ -152,7 +159,7 @@ export function backendChanged(backend, pathPrefix, changedFiles) {
 
   // Fork backends reuse backend/cpp/llama-cpp sources via thin wrappers;
   // changes to either directory must retrigger their pipelines.
-  return (backend === "turboquant" || backend === "bonsai") &&
+  return (backend === "turboquant" || backend === "cachyllama" || backend === "bonsai") &&
     changedFiles.some(file => file.startsWith("backend/cpp/llama-cpp/"));
 }
 
@@ -160,12 +167,13 @@ export function backendChanged(backend, pathPrefix, changedFiles) {
 // without it is a Python backend (see .github/backend-matrix.yml).
 const isDarwinPython = item => !item.lang;
 
-// backend_build_darwin.yml routes llama-cpp, ds4, privacy-filter and audio-cpp
+// backend_build_darwin.yml routes llama-cpp, cachyllama, ds4, privacy-filter and audio-cpp
 // to their own bespoke make targets; every other lang=go entry goes through
 // `make build-darwin-go-backend` -> scripts/build/golang-darwin.sh. Keep this
 // set in sync with the `if:` conditions in that workflow.
 const DARWIN_BESPOKE_BUILDERS = new Set([
   "llama-cpp",
+  "cachyllama",
   "ds4",
   "privacy-filter",
   "audio-cpp",
@@ -428,6 +436,16 @@ export const SHARED_BUILD_INPUTS = [
     linux: never,
     darwin: item => item.backend === "llama-cpp",
   },
+  {
+    matches: file => file === "scripts/build/cachyllama-darwin.sh",
+    linux: never,
+    darwin: item => item.backend === "cachyllama",
+  },
+  {
+    matches: file => file === ".docker/cachyllama-compile.sh",
+    linux: item => item.backend === "cachyllama",
+    darwin: never,
+  },
   {
     matches: file => file === "scripts/build/ds4-darwin.sh",
     linux: never,
diff --git a/scripts/lib/backend-filter_test.mjs b/scripts/lib/backend-filter_test.mjs
index 25d942ae642d..65f002125c03 100644
--- a/scripts/lib/backend-filter_test.mjs
+++ b/scripts/lib/backend-filter_test.mjs
@@ -82,6 +82,12 @@ const includes = [
     "tag-suffix": "-cpu-audio-cpp",
     "base-image": "ubuntu:24.04",
   },
+  {
+    backend: "cachyllama",
+    dockerfile: "./backend/Dockerfile.cachyllama",
+    "tag-suffix": "-cachyllama",
+    "base-image": "ubuntu:24.04",
+  },
 ];
 
 const includesDarwin = [
@@ -89,6 +95,7 @@ const includesDarwin = [
   { backend: "mlx", "tag-suffix": "-metal-darwin-arm64-mlx", "build-type": "mps" },
   { backend: "whisper", lang: "go", "tag-suffix": "-metal-darwin-arm64-whisper", "build-type": "metal" },
   { backend: "llama-cpp", lang: "go", "tag-suffix": "-metal-darwin-arm64-llama-cpp", "build-type": "metal" },
+  { backend: "cachyllama", lang: "go", "tag-suffix": "-metal-darwin-arm64-cachyllama", "build-type": "metal" },
   { backend: "ds4", lang: "go", "tag-suffix": "-metal-darwin-arm64-ds4", "build-type": "metal" },
 ];
 
@@ -206,6 +213,19 @@ test("a bespoke Darwin build script rebuilds only its own backend", () => {
   assert.deepEqual(names(filteredDarwin), ["ds4"]);
 });
 
+test("the CachyLLaMA Darwin build script rebuilds only CachyLLaMA", () => {
+  const { filteredDarwin } = run(["scripts/build/cachyllama-darwin.sh"]);
+
+  assert.deepEqual(names(filteredDarwin), ["cachyllama"]);
+});
+
+test("the CachyLLaMA compile helper rebuilds only Linux CachyLLaMA", () => {
+  const { filtered, filteredDarwin } = run([".docker/cachyllama-compile.sh"]);
+
+  assert.deepEqual(names(filtered), ["cachyllama"]);
+  assert.deepEqual(filteredDarwin, []);
+});
+
 test("an unclassified scripts/build/ file conservatively rebuilds everything", () => {
   const { filtered, filteredDarwin } = run([
     "scripts/build/package-something-new.sh",
@@ -227,7 +247,17 @@ test("tests for the packaging scripts do not rebuild anything", () => {
 test("turboquant still retriggers on llama-cpp source changes", () => {
   const { filtered } = run(["backend/cpp/llama-cpp/grpc-server.cpp"]);
 
-  assert.deepEqual(names(filtered), ["llama-cpp", "turboquant"]);
+  assert.deepEqual(names(filtered), ["cachyllama", "llama-cpp", "turboquant"]);
+});
+
+test("cachyllama maps to its wrapper source directory", () => {
+  assert.equal(
+    inferBackendPath({
+      backend: "cachyllama",
+      dockerfile: "./backend/Dockerfile.cachyllama",
+    }),
+    "backend/cpp/cachyllama/"
+  );
 });
 
 // ---------------------------------------------------------------------------

From 59281e4357421a18605034331cb1fd25768fe87a Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@localai.io>
Date: Thu, 30 Jul 2026 09:06:50 +0000
Subject: [PATCH 2/7] fix(cachyllama): use portable ARM build

Avoid the CPU_ALL_VARIANTS SME matrix on ARM, where the Linux and Darwin backend toolchains fail to compile CachyLLaMA. Keep the existing variant build for x86 and use the fully linked fallback build for ARM.

Assisted-by: Codex:gpt-5 [systematic-debugging]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
---
 .docker/cachyllama-compile.sh      | 12 +++++++-----
 scripts/build/cachyllama-darwin.sh | 25 +++++--------------------
 2 files changed, 12 insertions(+), 25 deletions(-)

diff --git a/.docker/cachyllama-compile.sh b/.docker/cachyllama-compile.sh
index 3d4c9d2e728f..6e5ee2e40456 100755
--- a/.docker/cachyllama-compile.sh
+++ b/.docker/cachyllama-compile.sh
@@ -20,13 +20,15 @@ fi
 cd /LocalAI/backend/cpp/cachyllama
 
 if [ -z "${BUILD_TYPE:-}" ]; then
-  # Pure CPU image: one ggml CPU_ALL_VARIANTS build replaces the per-microarch binaries.
-  # arm64: the armv9.2 SME variants need gcc-14 (gcc-13 rejects +sme).
+  # Keep arm64 on the portable, fully linked build. CachyLLaMA's ARM
+  # CPU_ALL_VARIANTS build includes SME variants that do not build reliably
+  # across the Linux and Darwin toolchains used by backend CI.
   if [ "${TARGETARCH}" = "arm64" ]; then
-    apt-get update -qq && apt-get install -y -qq gcc-14 g++-14
-    export CC=gcc-14 CXX=g++-14
+    make cachyllama-fallback
+  else
+    # One ggml CPU_ALL_VARIANTS build replaces the per-microarch x86 binaries.
+    make cachyllama-cpu-all
   fi
-  make cachyllama-cpu-all
 else
   # GPU build (cublas/hipblas/sycl/vulkan/...): single fallback CPU build, the accelerator
   # does the compute. Keeps the GPU compile from also building the CPU variant matrix and
diff --git a/scripts/build/cachyllama-darwin.sh b/scripts/build/cachyllama-darwin.sh
index 3e024607220c..898e5402471c 100755
--- a/scripts/build/cachyllama-darwin.sh
+++ b/scripts/build/cachyllama-darwin.sh
@@ -6,11 +6,10 @@ IMAGE_NAME="${IMAGE_NAME:-localai/cachyllama-darwin}"
 
 pushd backend/cpp/cachyllama
 
-# Single build via ggml CPU_ALL_VARIANTS: one binary plus the per-microarch Apple/arm
-# dylibs (apple_m1/m2_m3/m4, armv8.x) that ggml selects at runtime. GGML_METAL stays ON
-# and --target ggml also builds ggml-metal (via add_dependencies), so the Metal GPU
-# backend is still produced as a loadable libggml-metal.dylib.
-make cachyllama-cpu-all && \
+# CachyLLaMA's ARM CPU_ALL_VARIANTS build includes SME variants that do not build
+# reliably across the Darwin toolchains used by backend CI. The fallback build
+# remains Metal-enabled on Darwin and is fully linked.
+make cachyllama-fallback && \
 make cachyllama-grpc && \
 make cachyllama-rpc-server
 
@@ -20,24 +19,10 @@ mkdir -p build/darwin
 mkdir -p backend-images
 mkdir -p build/darwin/lib
 
-cp -rf backend/cpp/cachyllama/cachyllama-cpu-all build/darwin/
+cp -rf backend/cpp/cachyllama/cachyllama-fallback build/darwin/
 cp -rf backend/cpp/cachyllama/cachyllama-grpc build/darwin/
 cp -rf backend/cpp/cachyllama/cachyllama-rpc-server build/darwin/
 
-# Distribute the shared ggml/llama libraries from the CPU_ALL_VARIANTS build. Unlike the
-# old fully-static fallback build, these have @rpath install names, so the otool loop below
-# (which only copies deps that exist on disk) will not pick them up. The split is by suffix:
-#  - ggml emits its loadable backends (per-microarch CPU variants, metal, blas) with a .so
-#    suffix EVEN ON DARWIN. These go in the package ROOT next to the binary, because darwin
-#    run.sh execs the binary directly (no bundled ld.so) so ggml's executable-directory
-#    scan looks there.
-#  - the core libraries (libggml-base/libggml/libllama/libllama-common/libmtmd) use the
-#    platform .dylib suffix and are NEEDED deps; they go in lib/, resolved at load time via
-#    the DYLD_LIBRARY_PATH=lib that run.sh exports. -a preserves the version symlinks.
-SHLIBS=backend/cpp/cachyllama/ggml-shared-libs
-cp -a $SHLIBS/*.so build/darwin/
-cp -a $SHLIBS/*.dylib build/darwin/lib/
-
 # Set default additional libs only for Darwin on M chips (arm64)
 if [[ "$(uname -s)" == "Darwin" && "$(uname -m)" == "arm64" ]]; then
     ADDITIONAL_LIBS=${ADDITIONAL_LIBS:-$(ls /opt/homebrew/Cellar/protobuf/**/lib/libutf8_validity*.dylib 2>/dev/null)}

From c9225fe78b647d43d8674317bdebf0fc8be9ed34 Mon Sep 17 00:00:00 2001
From: Ettore Di Giacinto <mudler@localai.io>
Date: Fri, 31 Jul 2026 03:06:34 +0000
Subject: [PATCH 3/7] fix(cachyllama): link SSD cache implementation

CachyLLaMA splits persistent prompt-cache support into separate source files. Include those implementations in the monolithic LocalAI gRPC adapter when the fork provides them so the ARM64 fallback build resolves the page-manager symbols.

Assisted-by: Codex:gpt-5 [Codex]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
---
 backend/cpp/llama-cpp/grpc-server.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp
index 21e556beff8b..e96e2ad86ff9 100644
--- a/backend/cpp/llama-cpp/grpc-server.cpp
+++ b/backend/cpp/llama-cpp/grpc-server.cpp
@@ -49,6 +49,15 @@
 #include "decision_compat.h"
 #include "decision_images.h"
 #endif
+// CachyLLaMA splits its persistent prompt-cache implementation out of
+// server-context.cpp. Pull both implementation files into this monolithic
+// adapter TU when present; older llama.cpp pins do not ship them.
+#if __has_include("server-context-ssd-cache.cpp")
+#include "server-context-ssd-cache.cpp"
+#endif
+#if __has_include("server-context-page-manager.cpp")
+#include "server-context-page-manager.cpp"
+#endif
 #include "server-context.cpp"
 
 // LocalAI

From 5bee6872b4e6b72d7f71f0b7c94f38915ec00596 Mon Sep 17 00:00:00 2001
From: localai-org-maint-bot
 <306269227+localai-org-maint-bot@users.noreply.github.com>
Date: Mon, 28 Sep 2026 03:05:46 +0000
Subject: [PATCH 4/7] fix(ui): open options in the CachyLLaMA test

Use the existing helper so the test reaches its backend preference
assertions instead of failing with a ReferenceError.

Assisted-by: Codex:gpt-6
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
---
 core/http/react-ui/e2e/import-form-ux-batch-d.spec.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/http/react-ui/e2e/import-form-ux-batch-d.spec.js b/core/http/react-ui/e2e/import-form-ux-batch-d.spec.js
index cdc7baded1c9..38f1a88c0c43 100644
--- a/core/http/react-ui/e2e/import-form-ux-batch-d.spec.js
+++ b/core/http/react-ui/e2e/import-form-ux-batch-d.spec.js
@@ -93,7 +93,7 @@ test.describe('Import form UX — Batch D (progressive disclosure)', () => {
   })
 
   test('D1 — selecting cachyllama shows GGUF Quantizations + MMProj', async ({ page }) => {
-    await enterPowerPreferences(page)
+    await openOptions(page)
     await selectBackend(page, 'cachyllama')
     await expect(quantizationsInput(page)).toBeVisible()
     await expect(mmprojInput(page)).toBeVisible()

From b07214ee6d5a89b8e0f58258dddd9eab12c979ca Mon Sep 17 00:00:00 2001
From: localai-org-maint-bot
 <306269227+localai-org-maint-bot@users.noreply.github.com>
Date: Fri, 23 Oct 2026 10:04:43 +0000
Subject: [PATCH 5/7] fix(mcp): update assistant test client stub

Implement InstallLibraryBackend so the assistant tests satisfy the
LocalAIClient interface after the library integration.

Assisted-by: Codex:GPT-6
---
 core/http/endpoints/mcp/localai_assistant_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/http/endpoints/mcp/localai_assistant_test.go b/core/http/endpoints/mcp/localai_assistant_test.go
index 36407565b198..e0c59ac88bb5 100644
--- a/core/http/endpoints/mcp/localai_assistant_test.go
+++ b/core/http/endpoints/mcp/localai_assistant_test.go
@@ -79,6 +79,10 @@ func (stubClient) InstallBackend(_ context.Context, _ localaitools.InstallBacken
 	return "stub-backend-job", nil
 }
 
+func (stubClient) InstallLibraryBackend(_ context.Context, _ localaitools.InstallLibraryBackendRequest) (*localaitools.InstallLibraryBackendResult, error) {
+	return &localaitools.InstallLibraryBackendResult{Status: "installed", Name: "stub-backend", SourceID: "stub-source"}, nil
+}
+
 func (stubClient) UpgradeBackend(_ context.Context, _ string) (string, error) {
 	return "stub-upgrade-job", nil
 }

From 74a5717146cb27301240784eac1f953d78c60038 Mon Sep 17 00:00:00 2001
From: localai-org-maint-bot
 <306269227+localai-org-maint-bot@users.noreply.github.com>
Date: Fri, 23 Oct 2026 10:07:31 +0000
Subject: [PATCH 6/7] test(gallery): count current library backends

The shipped library grows when a backend is added. Compare the reader
against the current library instead of the frozen migration fixture.

Assisted-by: Codex:GPT-6
---
 core/gallery/backend_library_generated_test.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/core/gallery/backend_library_generated_test.go b/core/gallery/backend_library_generated_test.go
index 57676a6b3aaf..564f90982433 100644
--- a/core/gallery/backend_library_generated_test.go
+++ b/core/gallery/backend_library_generated_test.go
@@ -104,8 +104,12 @@ func TestLibraryGalleryReadsTheBackendLibrary(t *testing.T) {
 	official.Library = true
 	official.URL = srv.URL + "/backends/index.yaml"
 	backends := names(official)
-	if len(backends) != goldenGalleryNames {
-		t.Fatalf("the library gave %d backends, want %d", len(backends), goldenGalleryNames)
+	expected, err := BackendsFromLibrary(loadRealBackendLibrary(t))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(backends) != len(expected) {
+		t.Fatalf("the library gave %d backends, want %d", len(backends), len(expected))
 	}
 	llama := backends.FindByName("llama-cpp")
 	if llama == nil || !llama.IsMeta() || llama.CapabilitiesMap["nvidia"] != "cuda12-llama-cpp" || llama.Alias != "llama-cpp" {

From 46be469d88e8d9333b150062be46c432f1049ec3 Mon Sep 17 00:00:00 2001
From: localai-org-maint-bot
 <306269227+localai-org-maint-bot@users.noreply.github.com>
Date: Fri, 23 Oct 2026 14:06:15 +0000
Subject: [PATCH 7/7] test(gallery): use Gomega for backend counts

Keep the library count assertions within the test lint policy.

Assisted-by: Codex:gpt-6
---
 core/gallery/backend_library_generated_test.go | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/core/gallery/backend_library_generated_test.go b/core/gallery/backend_library_generated_test.go
index 564f90982433..2e088aaa05c0 100644
--- a/core/gallery/backend_library_generated_test.go
+++ b/core/gallery/backend_library_generated_test.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/pkg/system"
+	"github.com/onsi/gomega"
 	"gopkg.in/yaml.v3"
 )
 
@@ -105,12 +106,9 @@ func TestLibraryGalleryReadsTheBackendLibrary(t *testing.T) {
 	official.URL = srv.URL + "/backends/index.yaml"
 	backends := names(official)
 	expected, err := BackendsFromLibrary(loadRealBackendLibrary(t))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(backends) != len(expected) {
-		t.Fatalf("the library gave %d backends, want %d", len(backends), len(expected))
-	}
+	g := gomega.NewWithT(t)
+	g.Expect(err).To(gomega.Succeed())
+	g.Expect(backends).To(gomega.HaveLen(len(expected)))
 	llama := backends.FindByName("llama-cpp")
 	if llama == nil || !llama.IsMeta() || llama.CapabilitiesMap["nvidia"] != "cuda12-llama-cpp" || llama.Alias != "llama-cpp" {
 		t.Fatalf("the meta backend lost its body: %+v", llama)