fix(nvvm): cast bit-counting intrinsic results back to the return width

crates/rustc_codegen_nvvm/src/intrinsic.rs

@@ -605,7 +605,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 }
                 let (size, signed) = ty.int_size_and_signed(self.tcx);
                 let width = size.bits();
-                if name == sym::saturating_add || name == sym::saturating_sub {
+                let llret = if name == sym::saturating_add || name == sym::saturating_sub {
                     saturating_intrinsic_impl(
                         self,
                         width as u32,
@@ -673,7 +673,18 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                         }
                         _ => unreachable!(),
                     }
-                }
+                };
+                // `ctpop`, `ctlz`, `cttz` and their `_nonzero` variants are declared in `core` as
+                // returning `u32` for *every* operand width, while the corresponding LLVM
+                // intrinsics return a value as wide as their operand. Without this cast the
+                // oversized value is stored into the 4-byte result slot, where it is either
+                // dropped as an out-of-bounds store (`u64`/`u128`, yielding a constant `0`) or
+                // leaves the high bytes uninitialized (`u8`/`u16`).
+                //
+                // Casting to an identical type is a no-op, so the intrinsics in this arm that do
+                // return the operand width (`bswap`, `bitreverse`, `rotate_*`, `saturating_*`)
+                // are unaffected, and any intrinsic added here later is covered automatically.
+                self.intcast(llret, result.layout.llvm_type(self), false)
             }
             sym::raw_eq => {
                 use rustc_codegen_ssa::common::IntPredicate;

guide/src/features.md

@@ -28,7 +28,7 @@ around to adding it yet.
 | Match | ✔️ |
 | Proc Macros | ✔️ |
 | Try (`?`) | ✔️ |
-| 128 bit integers | 🟨 | Basic ops should work (and are emulated), advanced intrinsics like `ctpop`, `rotate`, etc are unsupported. |
+| 128 bit integers | 🟨 | Basic ops should work (and are emulated). The bit-manipulation intrinsics (`count_ones`, `leading_zeros`, `trailing_zeros`, `swap_bytes`, `reverse_bits`, `rotate_left`/`rotate_right`) are also emulated, over the two 64-bit halves. |
 | Unions | ✔️ |
 | Iterators | ✔️ |
 | Dynamic Dispatch | ✔️ |

tests/compiletests/ui/dis/count_ones_u128.rs

@@ -0,0 +1,18 @@
+// build-pass
+// compile-flags: -Cllvm-args=--disassemble-entry=count_ones_u128 --error-format=human -Cdebuginfo=0
+
+// Regression test for the emulated 128-bit path: `handle_128_bit_intrinsic` returns an `i128`,
+// which must be truncated to the `u32` that `core::intrinsics::ctpop` is declared to return.
+// Without the truncation the 16-byte store into the 4-byte result slot is discarded and both
+// `popc`s are eliminated as dead, so `u128::count_ones` silently returns `0` on device.
+//
+// The kernel below must contain two `popc.b64`s and the `add` that combines them.
+
+use cuda_std::kernel;
+
+#[kernel]
+pub unsafe fn count_ones_u128(buffer: *const u128, out: *mut u32) {
+    unsafe {
+        *out = (*buffer).count_ones();
+    }
+}

tests/compiletests/ui/dis/count_ones_u128.stderr

@@ -0,0 +1,25 @@
+.visible .entry count_ones_u128(
+    .param .u64 count_ones_u128_param_0,
+    .param .u64 count_ones_u128_param_1
+)
+{
+    .reg .b32     %r<3>;
+    .reg .b64     %rd<12>;
+
+
+    ld.param.u64     %rd1, [count_ones_u128_param_0];
+    ld.param.u64     %rd2, [count_ones_u128_param_1];
+    cvta.to.global.u64     %rd3, %rd2;
+    cvta.to.global.u64     %rd4, %rd1;
+    ld.global.v2.u64     {%rd5, %rd6}, [%rd4];
+    popc.b64     %r1, %rd5;
+    cvt.u64.u32     %rd9, %r1;
+    popc.b64     %r2, %rd6;
+    cvt.u64.u32     %rd10, %r2;
+    add.s64     %rd11, %rd9, %rd10;
+    st.global.u32     [%rd3], %rd11;
+    ret;
+
+}
+
+

tests/compiletests/ui/dis/count_ones_u64.rs

@@ -0,0 +1,18 @@
+// build-pass
+// compile-flags: -Cllvm-args=--disassemble-entry=count_ones_u64 --error-format=human -Cdebuginfo=0
+
+// Regression test: `core::intrinsics::ctpop` returns `u32` for every operand width, so the
+// `i64` result of `llvm.ctpop.i64` has to be truncated before it is stored into the `u32`
+// result slot. Without that truncation the oversized store is dropped and the `popc` is
+// eliminated as dead, so `count_ones` silently returns `0` on device.
+//
+// The kernel below must contain a `popc.b64`.
+
+use cuda_std::kernel;
+
+#[kernel]
+pub unsafe fn count_ones_u64(buffer: *const u64, out: *mut u32) {
+    unsafe {
+        *out = (*buffer).count_ones();
+    }
+}

tests/compiletests/ui/dis/count_ones_u64.stderr

@@ -0,0 +1,22 @@
+.visible .entry count_ones_u64(
+    .param .u64 count_ones_u64_param_0,
+    .param .u64 count_ones_u64_param_1
+)
+{
+    .reg .b32     %r<2>;
+    .reg .b64     %rd<7>;
+
+
+    ld.param.u64     %rd1, [count_ones_u64_param_0];
+    ld.param.u64     %rd2, [count_ones_u64_param_1];
+    cvta.to.global.u64     %rd3, %rd2;
+    cvta.to.global.u64     %rd4, %rd1;
+    ld.global.u64     %rd5, [%rd4];
+    popc.b64     %r1, %rd5;
+    cvt.u64.u32     %rd6, %r1;
+    st.global.u32     [%rd3], %rd6;
+    ret;
+
+}
+
+

tests/compiletests/ui/dis/leading_zeros_u64.rs

@@ -0,0 +1,18 @@
+// build-pass
+// compile-flags: -Cllvm-args=--disassemble-entry=leading_zeros_u64 --error-format=human -Cdebuginfo=0
+
+// Regression test: like `ctpop`, `core::intrinsics::ctlz` returns `u32` for every operand
+// width, so the `i64` result of `llvm.ctlz.i64` has to be truncated before it is stored into
+// the `u32` result slot. Without that truncation the oversized store is dropped and the
+// `clz` is eliminated as dead, so `leading_zeros` silently returns `0` on device.
+//
+// The kernel below must contain a `clz.b64`.
+
+use cuda_std::kernel;
+
+#[kernel]
+pub unsafe fn leading_zeros_u64(buffer: *const u64, out: *mut u32) {
+    unsafe {
+        *out = (*buffer).leading_zeros();
+    }
+}

tests/compiletests/ui/dis/leading_zeros_u64.stderr

@@ -0,0 +1,22 @@
+.visible .entry leading_zeros_u64(
+    .param .u64 leading_zeros_u64_param_0,
+    .param .u64 leading_zeros_u64_param_1
+)
+{
+    .reg .b32     %r<2>;
+    .reg .b64     %rd<7>;
+
+
+    ld.param.u64     %rd1, [leading_zeros_u64_param_0];
+    ld.param.u64     %rd2, [leading_zeros_u64_param_1];
+    cvta.to.global.u64     %rd3, %rd2;
+    cvta.to.global.u64     %rd4, %rd1;
+    ld.global.u64     %rd5, [%rd4];
+    clz.b64     %r1, %rd5;
+    cvt.u64.u32     %rd6, %r1;
+    st.global.u32     [%rd3], %rd6;
+    ret;
+
+}
+
+