From c0d11e78af92ab7748a7e89ae20f12d192fd7c6b Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Sun, 19 Jul 2026 03:58:34 +0530 Subject: [PATCH 1/2] fix: allow mixed turbo/q8_0 KV cache types in CUDA flash-attention selection Without GGML_CUDA_FA_ALL_QUANTS (the default), kernel selection rejected any K/V type mismatch. A turbo KV setup is almost always mixed (ctk q8_0 + ctv turbo2/3), so the flash-attention probe failed, FA was auto-disabled, and context creation either refused the quantized V cache outright or fell back to the non-FA path whose KQ buffer (~8.7 GiB at 64k ctx, ubatch 1024) OOMs at load. Allow exactly the mixed pairs that have FA template instances compiled in: turbo2/3/4 against each other, q8_0, and f16. Also gate turbo head-dim geometry (multiples of 64; 128 for turbo4) so shapes without a compiled instance report unsupported instead of asserting at dispatch. Forcing -fa on masked the bug (the probe is skipped); server deployments use fa auto and hit it on every load. --- ggml/src/ggml-cuda/fattn.cu | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 09eb6ff04e65..777f7063a49e 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -469,10 +469,34 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const #ifndef GGML_CUDA_FA_ALL_QUANTS if (K->type != V->type) { - return BEST_FATTN_KERNEL_NONE; + // Allow the mixed KV type pairs that have FA template instances compiled in: + // every turbo2/3/4 combination with q8_0 or f16 (turbo KV cache work). + auto is_turbo = [](ggml_type t) { + return t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0 || t == GGML_TYPE_TURBO4_0; + }; + auto is_turbo_compat = [&](ggml_type t) { + return is_turbo(t) || t == GGML_TYPE_Q8_0 || t == GGML_TYPE_F16; + }; + if (!(is_turbo(K->type) || is_turbo(V->type)) || + !is_turbo_compat(K->type) || !is_turbo_compat(V->type)) { + return BEST_FATTN_KERNEL_NONE; + } } #endif // GGML_CUDA_FA_ALL_QUANTS + // turbo kernels are instantiated only for head dims that are multiples of 64 + // (turbo4: multiples of 128, matching its block size). + { + auto turbo_geom_ok = [](ggml_type t, int64_t ne0) { + if (t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0) return ne0 % 64 == 0; + if (t == GGML_TYPE_TURBO4_0) return ne0 % 128 == 0; + return true; + }; + if (!turbo_geom_ok(K->type, K->ne[0]) || !turbo_geom_ok(V->type, V->ne[0])) { + return BEST_FATTN_KERNEL_NONE; + } + } + if (!ggml_cuda_fattn_kv_type_supported(K->type) || !ggml_cuda_fattn_kv_type_supported(V->type)) { return BEST_FATTN_KERNEL_NONE; } From 781726ddc0f911b3b197a6d1f02b4ca93eaa9818 Mon Sep 17 00:00:00 2001 From: Anirban Kar Date: Sun, 19 Jul 2026 04:00:28 +0530 Subject: [PATCH 2/2] feat: accept turbo2/3/4 cache type names in llama-bench The common arg parser already accepts them for llama-completion and llama-server; llama-bench has its own name table and rejected -ctk/-ctv turbo values. --- tools/llama-bench/llama-bench.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index 0756893881ff..a099bddbec65 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -500,6 +500,15 @@ static ggml_type ggml_type_from_name(const std::string & s) { if (s == "iq4_nl") { return GGML_TYPE_IQ4_NL; } + if (s == "turbo2") { + return GGML_TYPE_TURBO2_0; + } + if (s == "turbo3") { + return GGML_TYPE_TURBO3_0; + } + if (s == "turbo4") { + return GGML_TYPE_TURBO4_0; + } return GGML_TYPE_COUNT; }