From 66c4270c14f46ffc3bba6256cac7482ef9c44afe Mon Sep 17 00:00:00 2001 From: sstamenk Date: Sun, 19 Jul 2026 02:09:31 +0200 Subject: [PATCH 1/4] fix(rocm): avoid SIMT workgroup-count resonance Add one bounds-checked workgroup when a large HIP SIMT grid is an exact multiple of 256, avoiding scheduler underutilization observed on RDNA3 and RDNA4.\n\nCo-Authored-By: Claude --- csrc/gemm_4bit_simt.cu | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/csrc/gemm_4bit_simt.cu b/csrc/gemm_4bit_simt.cu index d4f34adb4..41e8080cc 100644 --- a/csrc/gemm_4bit_simt.cu +++ b/csrc/gemm_4bit_simt.cu @@ -490,7 +490,14 @@ void launch_gemm_4bit_simt( bnb_stream_t stream // CUDA/HIP stream // clang-format on ) { - const int n_blocks = (N + WARPS_PER_BLOCK - 1) / WARPS_PER_BLOCK; + int n_blocks = (N + WARPS_PER_BLOCK - 1) / WARPS_PER_BLOCK; +#if BNB_HIP + // Large HIP SIMT dispatches can underutilize the GPU when the workgroup + // count is an exact multiple of 256. One additional bounds-checked block + // breaks the scheduler resonance without changing useful work. + if (n_blocks >= 512 && n_blocks % 256 == 0) + n_blocks++; +#endif // M=1..8: M_BLOCK == M, so the inner m-loop fully unrolls at compile time. // M>8: M_BLOCK=8, ceil(M/8) grid rows. From 5675446a46563eefef7e79c211baa85cd42786d5 Mon Sep 17 00:00:00 2001 From: sstamenk Date: Mon, 20 Jul 2026 03:05:48 +0200 Subject: [PATCH 2/4] refactor(rocm): add explicit RDNA and CDNA macros Co-Authored-By: Claude --- csrc/common.cuh | 15 ++++++++++++++- csrc/gemm_4bit_simt.cu | 7 ++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/csrc/common.cuh b/csrc/common.cuh index 1490ced6e..02aa71569 100644 --- a/csrc/common.cuh +++ b/csrc/common.cuh @@ -4,10 +4,23 @@ #include "compat.cuh" +// AMD GPU architecture detection. Use explicit device macros rather than the +// broad __GFX9__/__GFX11__/__GFX12__ macros, which span distinct architectures. +#define IS_RDNA3 (defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__)) +#define IS_RDNA3_5 (defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__)) +#define IS_RDNA4 (defined(__gfx1200__) || defined(__gfx1201__)) +#define IS_RDNA (IS_RDNA3 || IS_RDNA3_5 || IS_RDNA4) + +#define IS_CDNA1 (defined(__gfx908__)) +#define IS_CDNA2 (defined(__gfx90a__)) +#define IS_CDNA3 (defined(__gfx942__)) +#define IS_CDNA4 (defined(__gfx950__)) +#define IS_CDNA (IS_CDNA1 || IS_CDNA2 || IS_CDNA3 || IS_CDNA4) + // Warp size #if BNB_HIP -#if defined(__GFX9__) +#if IS_CDNA #define BNB_WARP_SIZE 64 // CDNA #else #define BNB_WARP_SIZE 32 // RDNA diff --git a/csrc/gemm_4bit_simt.cu b/csrc/gemm_4bit_simt.cu index 41e8080cc..2cfb02842 100644 --- a/csrc/gemm_4bit_simt.cu +++ b/csrc/gemm_4bit_simt.cu @@ -4,10 +4,11 @@ #include #include +#include "common.cuh" #include "gemm_4bit_common.cuh" #include "gemm_4bit_simt.cuh" -#if defined(__GFX9__) +#if IS_CDNA // gfx9/CDNA tuning: // - use fmaf for fp32 accumulation // - accumulate fp16/bf16 pairs in fp32 via fused multiply-add @@ -26,9 +27,9 @@ #define BNB_SIMT_FP16_PACKED_ACCUM 0 #endif -// RDNA3/RDNA4 tuning: +// RDNA3/RDNA3.5/RDNA4 tuning: // - use native packed bf16/fp16 dot2 instructions with fp32 accumulation -#if defined(__GFX11__) || defined(__GFX12__) +#if IS_RDNA #define BNB_HIP_BF16_VDOT2 1 #define BNB_HIP_FP16_DOT2 1 #else From b06d03aa9ec1515520100f1026ecae4770b542fb Mon Sep 17 00:00:00 2001 From: sstamenk Date: Mon, 20 Jul 2026 13:51:40 +0200 Subject: [PATCH 3/4] refactor(rocm): detect RDNA2 architectures Co-Authored-By: Claude --- csrc/common.cuh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/csrc/common.cuh b/csrc/common.cuh index 02aa71569..9df046903 100644 --- a/csrc/common.cuh +++ b/csrc/common.cuh @@ -6,10 +6,13 @@ // AMD GPU architecture detection. Use explicit device macros rather than the // broad __GFX9__/__GFX11__/__GFX12__ macros, which span distinct architectures. +#define IS_RDNA2 \ + (defined(__gfx1030__) || defined(__gfx1031__) || defined(__gfx1032__) || defined(__gfx1033__) || \ + defined(__gfx1034__) || defined(__gfx1035__) || defined(__gfx1036__)) #define IS_RDNA3 (defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__)) #define IS_RDNA3_5 (defined(__gfx1150__) || defined(__gfx1151__) || defined(__gfx1152__) || defined(__gfx1153__)) #define IS_RDNA4 (defined(__gfx1200__) || defined(__gfx1201__)) -#define IS_RDNA (IS_RDNA3 || IS_RDNA3_5 || IS_RDNA4) +#define IS_RDNA (IS_RDNA2 || IS_RDNA3 || IS_RDNA3_5 || IS_RDNA4) #define IS_CDNA1 (defined(__gfx908__)) #define IS_CDNA2 (defined(__gfx90a__)) From 622dd290c5d6789bb16ff3f6fc578765bd9f6d51 Mon Sep 17 00:00:00 2001 From: sstamenk Date: Mon, 20 Jul 2026 14:55:45 +0200 Subject: [PATCH 4/4] fix(rocm): exclude RDNA2 from dot2 path Co-Authored-By: Claude --- csrc/gemm_4bit_simt.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csrc/gemm_4bit_simt.cu b/csrc/gemm_4bit_simt.cu index 2cfb02842..c0062c4f9 100644 --- a/csrc/gemm_4bit_simt.cu +++ b/csrc/gemm_4bit_simt.cu @@ -29,7 +29,7 @@ // RDNA3/RDNA3.5/RDNA4 tuning: // - use native packed bf16/fp16 dot2 instructions with fp32 accumulation -#if IS_RDNA +#if IS_RDNA3 || IS_RDNA3_5 || IS_RDNA4 #define BNB_HIP_BF16_VDOT2 1 #define BNB_HIP_FP16_DOT2 1 #else