diff --git a/include/xvec/detail/core.hpp b/include/xvec/detail/core.hpp index 1e92489..5e9d01b 100644 --- a/include/xvec/detail/core.hpp +++ b/include/xvec/detail/core.hpp @@ -401,7 +401,7 @@ template struct flags { namespace detail { class flag_convert {}; class flag_aligned {}; - template class flag_overaligned {}; + template class flag_overaligned {}; class flag_unchecked {}; // Add extra flags to the existing flags. @@ -421,7 +421,7 @@ inline constexpr flags<> flag_default{}; inline constexpr flags flag_convert{}; inline constexpr flags flag_aligned{}; -template requires (std::has_single_bit(_N)) inline constexpr flags> flag_overaligned{}; +template requires (std::has_single_bit(_N)) inline constexpr flags> flag_overaligned{}; /// @} cpp26_flags @@ -453,7 +453,7 @@ struct vec_traits static constexpr simd_size_type _numDataElements = complex_number<_Tp> ? 2 * _Np : _Np; /// The number of storage elements. This could be bigger than the number of data elements. - static constexpr simd_size_type _numStorageElements = isLlvm ? _numDataElements : std::bit_ceil((uint64_t)_numDataElements); + static constexpr simd_size_type _numStorageElements = isLlvm ? _numDataElements : std::bit_ceil((std::uint64_t)_numDataElements); /// The bare storage type used by the compiler for the elements. For /// arithmetic types this will typically be the same as the type itself, but diff --git a/include/xvec/detail/loadstore.hpp b/include/xvec/detail/loadstore.hpp index 58e2707..19d934a 100644 --- a/include/xvec/detail/loadstore.hpp +++ b/include/xvec/detail/loadstore.hpp @@ -10,6 +10,7 @@ #include +#include // std::size_t #include // std::contiguous_iterator, std::iter_value_t namespace _XVEC_NAMESPACE::simd @@ -146,7 +147,7 @@ partial_load(_It first, std::iter_difference_t<_It> n, /// @param f Memory-operation flags. template _Sent, typename... _Flags> constexpr detail::load_return_type_iter<_Vp, _It> partial_load(_It first, _Sent last, flags<_Flags...> f = {}) - { return partial_load<_Vp>(std::span(first, static_cast(last - first)), f); } + { return partial_load<_Vp>(std::span(first, static_cast(last - first)), f); } /// @brief Masked partial load from an iterator and sized sentinel. /// @param first The beginning of the source range. @@ -156,7 +157,7 @@ constexpr detail::load_return_type_iter<_Vp, _It> partial_load(_It first, _Sent template _Sent, typename... _Flags> constexpr detail::load_return_type_iter<_Vp, _It> partial_load(_It first, _Sent last, const detail::load_mask_type_iter<_Vp, _It>& m, flags<_Flags...> f = {}) - { return partial_load<_Vp>(std::span(first, static_cast(last - first)), m, f); } + { return partial_load<_Vp>(std::span(first, static_cast(last - first)), m, f); } /// @brief Unchecked load from a contiguous sized range. /// @param r The source range. @@ -199,7 +200,7 @@ unchecked_load(_It first, std::iter_difference_t<_It> n, /// @param f Memory-operation flags. template _Sent, typename... _Flags> constexpr detail::load_return_type_iter<_Vp, _It> unchecked_load(_It first, _Sent last, flags<_Flags...> f = {}) - { return partial_load<_Vp>(std::span{first, static_cast(last - first)}, detail::add_unchecked_flag(f)); } + { return partial_load<_Vp>(std::span{first, static_cast(last - first)}, detail::add_unchecked_flag(f)); } /// @brief Masked unchecked load from an iterator and sized sentinel. /// @param first The beginning of the source range. @@ -209,7 +210,7 @@ constexpr detail::load_return_type_iter<_Vp, _It> unchecked_load(_It first, _Sen template _Sent, typename... _Flags> constexpr detail::load_return_type_iter<_Vp, _It> unchecked_load(_It first, _Sent last, const detail::load_mask_type_iter<_Vp, _It>& m, flags<_Flags...> f = {}) - { return partial_load<_Vp>(std::span(first, static_cast(last - first)), m, detail::add_unchecked_flag(f)); } + { return partial_load<_Vp>(std::span(first, static_cast(last - first)), m, detail::add_unchecked_flag(f)); } /// @brief Store the elements from a basic_vec value into a contiguous range, /// performing a suitable conversion if necessary. Both unmasked and masked @@ -308,7 +309,7 @@ constexpr void partial_store(const basic_vec<_Tp, _Abi>& value, _It first, std:: template _Sent, typename... _Flags> requires std::indirectly_writable<_It, _Tp> constexpr void partial_store(const basic_vec<_Tp, _Abi>& value, _It first, _Sent last, flags<_Flags...> f = {}) - { partial_store(value, std::span(first, static_cast(last - first)), f); } + { partial_store(value, std::span(first, static_cast(last - first)), f); } /// @brief Masked partial store using an iterator and sized sentinel. /// @param value The source vector. @@ -320,7 +321,7 @@ template constexpr void partial_store(const basic_vec<_Tp, _Abi>& value, _It first, _Sent last, const typename basic_vec<_Tp, _Abi>::mask_type& m, flags<_Flags...> f = {}) - { partial_store(value, std::span(first, static_cast(last - first)), m, f); } + { partial_store(value, std::span(first, static_cast(last - first)), m, f); } /// @brief Unchecked store into a contiguous sized range. /// @param value The source vector. @@ -372,7 +373,7 @@ constexpr void unchecked_store(const basic_vec<_Tp, _Abi>& value, _It first, std template _Sent, typename... _Flags> requires std::indirectly_writable<_It, _Tp> constexpr void unchecked_store(const basic_vec<_Tp, _Abi>& value, _It first, _Sent last, flags<_Flags...> f = {}) - { partial_store(value, std::span(first, static_cast(last - first)), detail::add_unchecked_flag(f)); } + { partial_store(value, std::span(first, static_cast(last - first)), detail::add_unchecked_flag(f)); } /// @brief Masked unchecked store using an iterator and sized sentinel. /// @param value The source vector. @@ -384,7 +385,7 @@ template constexpr void unchecked_store(const basic_vec<_Tp, _Abi>& value, _It first, _Sent last, const typename basic_vec<_Tp, _Abi>::mask_type& m, flags<_Flags...> f = {}) - { partial_store(value, std::span(first, static_cast(last - first)), m, detail::add_unchecked_flag(f)); } + { partial_store(value, std::span(first, static_cast(last - first)), m, detail::add_unchecked_flag(f)); } /// @} cpp26_load_store diff --git a/include/xvec/detail/mask.hpp b/include/xvec/detail/mask.hpp index 912e522..9d82ff4 100644 --- a/include/xvec/detail/mask.hpp +++ b/include/xvec/detail/mask.hpp @@ -570,9 +570,9 @@ constexpr auto mask_from_count(simd_size_type n) noexcept { // CTAD, to convert a basic_mask into a basic_vec, equivalent to decltype(+mask). /// @brief Deduce a vector type from a mask, equivalent to decltype(+k). /// @ingroup cpp26_ctor -template +template basic_vec(basic_mask<_Bytes, _Abi> k) -> basic_vec::traits::element_signed_container_type, _Abi>; -} // namespace _XVEC_NAMESPACE::simd \ No newline at end of file +} // namespace _XVEC_NAMESPACE::simd diff --git a/include/xvec/detail/reduce.hpp b/include/xvec/detail/reduce.hpp index 792154a..faf2b53 100644 --- a/include/xvec/detail/reduce.hpp +++ b/include/xvec/detail/reduce.hpp @@ -59,7 +59,7 @@ inline _Tp reduce_helper(const basic_vec<_Tp, _Abi>& v, _BinaryFn binop) { // register. It is better to handle anything small as a complete register, // rather than extracting and inserting sub-register pieces in the final // stages of the reduction. - constexpr int halfSize = std::bit_ceil((uint64_t)_Size) / 2; + constexpr int halfSize = std::bit_ceil((std::uint64_t)_Size) / 2; constexpr auto opSize = std::max(register_size<_Tp, 1>, halfSize); const auto lower = permute(v, perm_uninitResize); diff --git a/include/xvec/detail/std_bit.hpp b/include/xvec/detail/std_bit.hpp index d5d95e4..c248b2b 100644 --- a/include/xvec/detail/std_bit.hpp +++ b/include/xvec/detail/std_bit.hpp @@ -171,9 +171,9 @@ constexpr _Vp bit_ceil(const _Vp& x) noexcept { // Normally bit_ceil would need to be wary of shifts, since some types get // promoted and would give the wrong result. This is't a problem for SIMD // operations since promotion doesn't occur. - constexpr auto one = _Vp(uint8_t(1)); + constexpr auto one = _Vp(std::uint8_t(1)); auto shift_amount = _Vp(bit_width(x - one)); - return select(x < uint8_t(2), one, one << shift_amount); + return select(x < std::uint8_t(2), one, one << shift_amount); } } @@ -190,7 +190,7 @@ constexpr _Vp bit_floor(const _Vp& v) noexcept { else { auto log2 = rebind_cast<_Tp>(detail::cw::digits - 1> - countl_zero(v)); - return select(v != _Vp(), uint8_t(1) << log2, _Vp()); + return select(v != _Vp(), std::uint8_t(1) << log2, _Vp()); } } diff --git a/include/xvec/detail/std_math.hpp b/include/xvec/detail/std_math.hpp index d2a949a..70c4f6b 100644 --- a/include/xvec/detail/std_math.hpp +++ b/include/xvec/detail/std_math.hpp @@ -9,6 +9,7 @@ #pragma once #include +#include // std::uint8_t #include // std::numeric_limits #include @@ -240,7 +241,7 @@ constexpr deduced_mask_t<_Vp> isnan(generic_tag, const _Vp& v) // ones, and the fraction nonzero. Checking a NaN against itself is another // way but is fragile with respect to compiler optimisations. const auto asC = vec_as_container(v); - return ((asC & expMask) == expMask) & ((asC & fractionMask) != uint8_t(0)); + return ((asC & expMask) == expMask) & ((asC & fractionMask) != std::uint8_t(0)); } /// Generic implementation of infinity classification. @@ -270,7 +271,7 @@ template constexpr detail::deduced_mask_t<_Vp> signbit(generic_tag, const _Vp& v) { constexpr auto signBit = vec_as_container(_Vp(typename _Vp::value_type(-0.0))); - auto m = (vec_as_container(v) & signBit) != uint8_t(0); + auto m = (vec_as_container(v) & signBit) != std::uint8_t(0); return detail::deduced_mask_t<_Vp>(m); } diff --git a/include/xvec/generic/generic_impl.hpp b/include/xvec/generic/generic_impl.hpp index 0733366..025b5e9 100644 --- a/include/xvec/generic/generic_impl.hpp +++ b/include/xvec/generic/generic_impl.hpp @@ -982,7 +982,7 @@ constexpr basic_vec<_Tp, _Abi> rotate_right(generic_tag, const basic_vec<_Tp, _A template requires (sizeof(typename _Vp::value_type) == sizeof(typename _Sp::value_type)) constexpr _Vp fsr(generic_tag, const _Vp& high, const _Vp& low, const _Sp& s) { - constexpr uint8_t numBits = sizeof(typename _Vp::value_type) * 8; + constexpr std::uint8_t numBits = sizeof(typename _Vp::value_type) * 8; const auto sm = _Vp(s) % numBits; const _Vp r = (low >> sm) | (high << ((numBits - sm) % numBits)); return select(sm != _Vp(), r, low); @@ -991,7 +991,7 @@ constexpr _Vp fsr(generic_tag, const _Vp& high, const _Vp& low, const _Sp& s) { template requires (sizeof(typename _Vp::value_type) == sizeof(typename _Sp::value_type)) constexpr _Vp fsl(generic_tag, const _Vp& high, const _Vp& low, const _Sp& s) { - constexpr uint8_t numBits = sizeof(typename _Vp::value_type) * 8; + constexpr std::uint8_t numBits = sizeof(typename _Vp::value_type) * 8; const auto sm = _Vp(s) % numBits; const _Vp r = (high << sm) | (low >> ((numBits - sm) % numBits)); return select(sm != _Vp(), r, high); @@ -1093,7 +1093,7 @@ constexpr basic_vec<_Tp, _Abi> byteswap(_Vendor, const basic_vec<_Tp, _Abi>& x) // elements). auto impl = [](auto piece) { - const auto asBytes = simd_bit_cast(piece); + const auto asBytes = simd_bit_cast(piece); const auto rev = permute(asBytes, [](simd_size_type idx) -> simd_size_type { return idx ^ (sizeof(_Tp) - 1); }); return simd_bit_cast<_Tp>(rev); }; @@ -1109,7 +1109,7 @@ template // :TODO: Should be unsigned_integral once bitswap is constexpr _Vp bitreverse(generic_tag, const _Vp& x) noexcept { auto doBrev = [](X bits) { - auto asBytes = simd_bit_cast(bits); + auto asBytes = simd_bit_cast(bits); using BM = decltype(asBytes); constexpr BM m1(cw<0x55u>); @@ -1178,7 +1178,7 @@ constexpr basic_vec<_Tp, _Abi> mask_to_vec(compact_mask_tag, const basic_mask<_B /// @internal template constexpr auto has_single_bit(generic_tag, const basic_vec<_Tp, _Abi>& x) noexcept { - return (x != uint8_t(0)) && ((x & (x - uint8_t(1))) == uint8_t(0)); + return (x != std::uint8_t(0)) && ((x & (x - std::uint8_t(1))) == std::uint8_t(0)); } /// @brief Perform a reduction to boolean on a mask value. The functions below are correct but very likely @@ -1189,7 +1189,7 @@ constexpr auto has_single_bit(generic_tag, const basic_vec<_Tp, _Abi>& x) noexce constexpr bool mask_all_of(generic_tag, const mask_type auto& mask) noexcept { bool allSet = true; // Assume until otherwise auto processBlock = [&](_S m) { - constexpr uint64_t allBits = (uint64_t(1) << _S::size) - 1; + constexpr std::uint64_t allBits = (std::uint64_t(1) << _S::size) - 1; allSet &= (allBits == m.to_ullong()); }; @@ -1207,7 +1207,7 @@ constexpr bool mask_any_of(generic_tag, const mask_type auto& mask) noexcept { // really big masks. Note also that it would be more efficient to OR the // values together as vecs and call to_ullong at the end, but the // compiler (clang/oneAPI) figures this out for itself anyway. - uint64_t reducedBits = 0; + std::uint64_t reducedBits = 0; chunked_invoke([&](auto m) { reducedBits |= m.to_ullong(); }, mask); return reducedBits != 0; } @@ -1222,7 +1222,7 @@ constexpr bool mask_none_of(generic_tag, const mask_type auto& mask) noexcept { // really big masks. Note also that it would be more efficient to OR the // values together as vecs and call to_ullong at the end, but the // compiler (clang/oneAPI) figures this out for itself anyway. - uint64_t reducedBits = 0; + std::uint64_t reducedBits = 0; chunked_invoke([&](auto m) { reducedBits |= m.to_ullong(); }, mask); return reducedBits == 0; } @@ -1277,13 +1277,13 @@ constexpr simd_size_type mask_reduce_min_index(compact_mask_tag, const _Mp& mask // limit of 128 bits anyway, so exploit that to make the code simpler. static_assert(_Mp::size <= 128, "This function only handles up to 128 bits"); - auto lowResult = std::countr_zero((uint64_t)mask.to_builtin()); + auto lowResult = std::countr_zero((std::uint64_t)mask.to_builtin()); if constexpr (_Mp::size <= 64) return lowResult; else { - auto highResult = std::countr_zero((uint64_t)(mask.to_builtin() >> 64)); + auto highResult = std::countr_zero((std::uint64_t)(mask.to_builtin() >> 64)); if (lowResult < 64) return lowResult; else return highResult + 64; } @@ -1315,13 +1315,13 @@ constexpr simd_size_type mask_reduce_max_index(compact_mask_tag, const _Mp& mask // limit of 128 bits anyway, so exploit that to make the code simpler. static_assert(_Mp::size <= 128, "This function only handles up to 128 bits"); - auto lowResult = 63 - std::countl_zero((uint64_t)mask.to_builtin()); + auto lowResult = 63 - std::countl_zero((std::uint64_t)mask.to_builtin()); if constexpr (_Mp::size <= 64) return lowResult; else { - auto upperBits = (uint64_t)(mask.to_builtin() >> 64); + auto upperBits = (std::uint64_t)(mask.to_builtin() >> 64); if (upperBits != 0) return 127 - std::countl_zero(upperBits); else @@ -1348,7 +1348,7 @@ constexpr auto get_n_bit_mask(generic_tag, const _Mp&, simd_size_type _n) { // mask means switching to unsigned bytes. Even then it can only go to 256 // elements, but for now this will do. auto unsignedIota = iota; - return unsignedIota < uint8_t(std::min(_n, 255)); + return unsignedIota < std::uint8_t(std::min(_n, 255)); } else { using _X = typename _Mp::traits::signed_vec_for_mask; return iota<_X> < (typename _X::value_type)std::min(_n, _Mp::size()); @@ -1550,7 +1550,7 @@ constexpr _Result generate_mask_from_unsigned(compact_mask_tag, std::unsigned_in template constexpr _Vp rcp(generic_tag, const _Vp& x) - { return uint8_t(1) / x; } + { return std::uint8_t(1) / x; } template constexpr _Vp sqrt(generic_tag, const _Vp& x) { @@ -1563,7 +1563,7 @@ constexpr _Vp sqrt(generic_tag, const _Vp& x) { template constexpr _Vp rsqrt(generic_tag, const _Vp& x) - { return uint8_t(1) / sqrt(x); } + { return std::uint8_t(1) / sqrt(x); } enum class RoundOp{CEIL, FLOOR, TRUNC, ROUND, RINT}; diff --git a/include/xvec/x86/intrin.hpp b/include/xvec/x86/intrin.hpp index a6bc50b..a5d63ad 100644 --- a/include/xvec/x86/intrin.hpp +++ b/include/xvec/x86/intrin.hpp @@ -59,7 +59,7 @@ concept zmm_register = (sizeof(_Vec) > 32 && sizeof(_Vec) <= 64) && (std::same_a /// @param b The input byte elements. /// @param m The matrix elements. /// @return The transformed byte elements. -template +template requires ((sizeof(typename _Bytes::value_type) == 1) && (sizeof(typename _Matrix::value_type) == 8) && ((_Matrix::size() * 8) == _Bytes::size())) @@ -79,7 +79,7 @@ inline _Bytes gf2p8affine(const _Bytes& b, const _Matrix& m) }; return std::bit_cast<_Bytes>(chunked_invoke([=](auto x, auto y) { return decltype(x)(impl(x, y)); }, - simd_bit_cast(b), simd_bit_cast(m))); + simd_bit_cast(b), simd_bit_cast(m))); } /// This variant of the GFNI affine instruction provides an easy way of @@ -90,13 +90,13 @@ inline _Bytes gf2p8affine(const _Bytes& b, const _Matrix& m) /// @param b The input byte elements. /// @param m The shared matrix. /// @return The transformed byte elements. -template +template requires (sizeof(typename _Bytes::value_type) == 1) // Allow any type of single byte element to be transformed. -inline _Bytes gf2p8affine(const _Bytes& b, uint64_t m) +inline _Bytes gf2p8affine(const _Bytes& b, std::uint64_t m) { constexpr int roundedTo8 = ((_Bytes::size() + 7) / 8) * 8; auto resizeTo8 = permute(b, detail::perm_uninitResize); - auto r = gf2p8affine(resizeTo8, vec(m)); + auto r = gf2p8affine(resizeTo8, vec(m)); return take<(int)_Bytes::size()>(r); } diff --git a/include/xvec/x86/permute.hpp b/include/xvec/x86/permute.hpp index 85b01c0..ba13735 100644 --- a/include/xvec/x86/permute.hpp +++ b/include/xvec/x86/permute.hpp @@ -24,8 +24,8 @@ namespace detail // It is no great pain then to use pdef, since most of the Intel AVX-512 // instructions will have that readily available. #if defined(__BMI2__) -inline uint64_t dupBits (uint32_t bits) { - auto spreadBits = _pdep_u64(uint64_t(bits), 0x55555555); +inline std::uint64_t dupBits (std::uint32_t bits) { + auto spreadBits = _pdep_u64(std::uint64_t(bits), 0x55555555); return spreadBits | (spreadBits << 1); } #endif @@ -59,7 +59,7 @@ constexpr auto permute_large_elements(const basic_vec<_Tp, _AbiT>& values, const }, indexes); // Now do the permute again using 64-bit elements with the new indexes. - auto r = permute(simd_bit_cast(values), bigIdxs); + auto r = permute(simd_bit_cast(values), bigIdxs); // Map back to the original 128-bit elements. return simd_bit_cast<_Tp>(r); @@ -76,7 +76,7 @@ constexpr auto permute_byte_shuffle (const basic_vec<_Tp, _AbiT>& v, const basic static_assert(basic_vec<_Tp, _AbiT>::size <= 16 && basic_vec<_Ip, _AbiI>::size <= 16, "Data and indexes must fit first lane"); // :TODO: In future both indexes and values could extend beyond lanes through // duplication or multi-lane index merging, which will be faster than using permutex[2]var_epi8. - return vec<_Tp, basic_vec<_Ip, _AbiI>::size>(_mm_shuffle_epi8(v.to_register(), (indexes & uint8_t(0xf)).to_register())); + return vec<_Tp, basic_vec<_Ip, _AbiI>::size>(_mm_shuffle_epi8(v.to_register(), (indexes & std::uint8_t(0xf)).to_register())); } /// Permute one register of Intel AVX2 values. @@ -291,7 +291,7 @@ permute(VENDOR tag, const basic_vec<_Tp, _AbiTp>& v, const basic_vec<_Ip, _AbiIp { // Special case - Intel AVX2 can't permute small element values so bump the size up to // something that can be supported. - const auto r = permute(rebind_cast(vc), indexes); + const auto r = permute(rebind_cast(vc), indexes); const auto resizeBack = rebind_cast<_C>(r); return simd_bit_cast<_Tp>(resizeBack); } @@ -341,13 +341,13 @@ constexpr auto generated_permute(x86_avx512_tag, const basic_mask<_Bytes, _Abi>& m, _Gp) { #if defined(__AVX512BMI__) - using _I = uint8_t; + using _I = std::uint8_t; #else // Not always a good idea to use 16-bit, as the instruction itself is quite // slow, but using 32-bit could lead to multiple registers which is also slow. // One special case is when there are fewer bits than a lane has bytes, in // which case epi8 could be used instead. - using _I = std::conditional_t::size <= 16, uint8_t, uint16_t>; + using _I = std::conditional_t::size <= 16, std::uint8_t, std::uint16_t>; #endif using _R = vec<_I, _OutSize>; @@ -404,19 +404,19 @@ inline target_overloads compress_one_register { #else // In the absence of __AVX512VBMI2__ instructions for 8/16-bit, synthesise them by using their 32-bit counterparts. - [] _Vec>(_Vec v, auto m, auto f) { + [] _Vec>(_Vec v, auto m, auto f) { auto compressed = _mm512_mask_compress_epi32(_mm512_cvtepi8_epi32(f.to_register()), m.to_register(), _mm512_cvtepi8_epi32(v.to_register())); return _Vec(_mm512_cvtepi32_epi8(compressed)); }, - [] _Vec>(_Vec v, auto m, auto f) { + [] _Vec>(_Vec v, auto m, auto f) { auto compressed = _mm256_mask_compress_epi32(_mm256_cvtepi16_epi32(f.to_register()), m.to_register(), _mm256_cvtepi16_epi32(v.to_register())); return _Vec(_mm256_cvtepi32_epi16(compressed)); }, - [] _Vec>(_Vec v, auto m, auto f) { + [] _Vec>(_Vec v, auto m, auto f) { auto compressed = _mm512_mask_compress_epi32(_mm512_cvtepi16_epi32(f.to_register()), m.to_register(), _mm512_cvtepi16_epi32(v.to_register())); @@ -487,7 +487,7 @@ compress_by_mask_snc(const _V& values, const typename _V::mask_type& selector, t // indexes, and then use that to permute the input. The output needs to be // masked to introduce zeros in unwritten elements. constexpr auto indexes = iota>; - const auto compressedIndexes = chunked_invoke(compress_one_register, indexes, mask(selector), vec()); + const auto compressedIndexes = chunked_invoke(compress_one_register, indexes, mask(selector), vec()); auto compressed = permute(values, compressedIndexes); auto numSetBits = reduce_count(selector); auto finalMask = _V::mask_type::__mask_from_count(numSetBits); @@ -540,7 +540,7 @@ compress_mask_by_mask(x86_avx2_tag, const _M& values, const _M& selector) // modify the code below to handle more chunks of data. static_assert(_Np <= 128); - using _R = std::conditional_t<(_Np > 64), __uint128_t, uint64_t>; + using _R = std::conditional_t<(_Np > 64), __uint128_t, std::uint64_t>; // Process up to the first 64 bits. constexpr int lowerN = std::min(_Np, 64); @@ -551,13 +551,13 @@ compress_mask_by_mask(x86_avx2_tag, const _M& values, const _M& selector) if constexpr (_Np > 64) { _R upperCompressed = _pext_u64(drop<64>(values).to_ullong(), drop<64>(selector).to_ullong()); - resultInt |= (upperCompressed << std::popcount((uint64_t)maskAsUll)); + resultInt |= (upperCompressed << std::popcount((std::uint64_t)maskAsUll)); } // The result is in an integer. Convert that back into a mask. - _M resultMask((uint64_t)resultInt); + _M resultMask((std::uint64_t)resultInt); if constexpr (_Np > 64) { - auto upper = resize_t<_Np - 64, _M>(uint64_t(resultInt >> 64)); + auto upper = resize_t<_Np - 64, _M>(std::uint64_t(resultInt >> 64)); resultMask = cat(take<64>(resultMask), upper); } @@ -580,13 +580,13 @@ compress_mask_by_mask(x86_avx512_tag, const _M& v, const _M& selector) // Process the first 64 bits. using _B = typename _M::builtin_type; - _B result = _pext_u64((uint64_t)vd, (uint64_t)md); + _B result = _pext_u64((std::uint64_t)vd, (std::uint64_t)md); // Process the top 64-bits too and then combine them together. if constexpr (_Np > 64) { _B upperCompressed = _pext_u64(vd >> 64, md >> 64); - result |= (upperCompressed << std::popcount((uint64_t)md)); + result |= (upperCompressed << std::popcount((std::uint64_t)md)); } return _M::from_builtin(result); @@ -616,17 +616,17 @@ inline target_overloads expand_one_register { // converting to a 4B type, expanding that, and converting back again. For the // expand instruction to have only one register to work it must accept no // more than 16 elements to start with (i.e., 128 or 256-bits respectively). - [] _Vec>(_Vec v, auto m, auto alt) { + [] _Vec>(_Vec v, auto m, auto alt) { auto compressed = _mm512_maskz_expand_epi32(m.to_register(), _mm512_cvtepi8_epi32(v.to_register())); return _Vec(_mm512_mask_cvtepi32_epi8(alt.to_register(), m.to_register(), compressed)); }, - [] _Vec>(_Vec v, auto m, auto alt) { + [] _Vec>(_Vec v, auto m, auto alt) { auto compressed = _mm256_maskz_expand_epi32(m.to_register(), _mm256_cvtepi16_epi32(v.to_register())); return _Vec(_mm256_mask_cvtepi32_epi16(alt.to_register(), m.to_register(), compressed)); }, - [] _Vec>(_Vec v, auto m, auto alt) { + [] _Vec>(_Vec v, auto m, auto alt) { auto compressed = _mm512_maskz_expand_epi32(m.to_register(), _mm512_cvtepi16_epi32(v.to_register())); return _Vec(_mm512_mask_cvtepi32_epi16(alt.to_register(), m.to_register(), compressed)); @@ -693,7 +693,7 @@ expand_by_mask_snc(const _V& value, const typename _V::mask_type& selector, cons // by expanding iota, and then permuting using the indexes. // :TODO: The permute should cope better with mixed sizes than it currently does. constexpr auto indexes = iota>; - const auto expandedIndexes = chunked_invoke(expand_one_register, indexes, mask(selector), vec()); + const auto expandedIndexes = chunked_invoke(expand_one_register, indexes, mask(selector), vec()); auto expanded = permute(value, expandedIndexes); return select(selector, expanded, original); } @@ -736,7 +736,7 @@ expand_mask_by_mask(x86_avx_tag, const _M& values, const _M& selector, const _M& // modify the code below to handle more chunks of data. static_assert(_Np <= 128); - using _R = std::conditional_t<(_Np > 64), __uint128_t, uint64_t>; + using _R = std::conditional_t<(_Np > 64), __uint128_t, std::uint64_t>; // Process the first 64 bits. constexpr int lowerN = std::min(_Np, 64); @@ -748,16 +748,16 @@ expand_mask_by_mask(x86_avx_tag, const _M& values, const _M& selector, const _M& if constexpr (_Np > 64) { _R upper64Value = drop<64>(values).to_ullong(); - uint64_t upper64Selector = drop<64>(selector).to_ullong(); + std::uint64_t upper64Selector = drop<64>(selector).to_ullong(); auto wholeValue = (upper64Value << 64) | _R(lowerValue); - auto bitsToExpand = uint64_t(wholeValue >> std::popcount(lowerSelector)); + auto bitsToExpand = std::uint64_t(wholeValue >> std::popcount(lowerSelector)); _R expandedBits = _pdep_u64(bitsToExpand, upper64Selector); resultInt |= _R(expandedBits) << 64; } - _M resultMask((uint64_t)resultInt); + _M resultMask((std::uint64_t)resultInt); if constexpr (_Np > 64) - resultMask = cat(take<64>(resultMask), resize_t<_Np - 64, _M>(uint64_t(resultInt >> 64))); + resultMask = cat(take<64>(resultMask), resize_t<_Np - 64, _M>(std::uint64_t(resultInt >> 64))); // Blend in any original bits and return the final result. auto final = resultMask.to_builtin() | (original.to_builtin() & ~(selector.to_builtin())); @@ -778,13 +778,13 @@ expand_mask_by_mask(x86_avx512_tag, const _M& values, const _M& selector, const static_assert(_Np <= 128); // Process the first 64 bits. - typename _M::builtin_type result = _pdep_u64((uint64_t)vd, (uint64_t)sd); + typename _M::builtin_type result = _pdep_u64((std::uint64_t)vd, (std::uint64_t)sd); // // If there are more than 64-bits, process the top 64-bits too and then combine them together. if constexpr (_Np > 64) { - uint64_t useBits = (uint64_t)(vd >> std::popcount((uint64_t)sd)); - typename _M::builtin_type expandedBits = _pdep_u64(useBits, (uint64_t)(sd >> 64)); + std::uint64_t useBits = (std::uint64_t)(vd >> std::popcount((std::uint64_t)sd)); + typename _M::builtin_type expandedBits = _pdep_u64(useBits, (std::uint64_t)(sd >> 64)); result |= expandedBits << 64; } @@ -868,13 +868,13 @@ constexpr auto permute_stride(x86_avx512_tag, const _Mp& in) noexcept { // currently 128-bits, which means a stride of 2 will never generate more than // 64 bits. constexpr auto bits = every_nth_bit<_NewMp, _Np>(); - uint64_t t = _pext_u64((uint64_t)iv, (uint64_t)bits); + std::uint64_t t = _pext_u64((std::uint64_t)iv, (std::uint64_t)bits); // Handle the upper 64-bits if they are also present. if constexpr (_OldSize > 64) { - auto upperBits = uint64_t(bits >> 64); - uint64_t b = _pext_u64((uint64_t)(iv >> 64), upperBits); - t |= b << std::popcount(uint64_t(bits & 0xFFFFFFFFFFFFFFFF)); + auto upperBits = std::uint64_t(bits >> 64); + std::uint64_t b = _pext_u64((std::uint64_t)(iv >> 64), upperBits); + t |= b << std::popcount(std::uint64_t(bits & 0xFFFFFFFFFFFFFFFF)); } return _NewMp::from_builtin(t); diff --git a/include/xvec/x86/x86_impl.hpp b/include/xvec/x86/x86_impl.hpp index ff0ea3e..2d84d43 100644 --- a/include/xvec/x86/x86_impl.hpp +++ b/include/xvec/x86/x86_impl.hpp @@ -505,20 +505,20 @@ constexpr void store_masked(x86_avx512_tag, const _Vp& value_original, std::span mandates_for_store(value_original, to, flags); target_overloads call_insn { - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi8(p, m, v.to_register()); }, - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi16(p, m, v.to_register()); }, - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi32(p, m, v.to_register()); }, - [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, m, v.to_register()); }, + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi8(p, m, v.to_register()); }, + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi16(p, m, v.to_register()); }, + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi32(p, m, v.to_register()); }, + [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, m, v.to_register()); }, - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi8(p, m, v.to_register()); }, - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi16(p, m, v.to_register()); }, - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi32(p, m, v.to_register()); }, - [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, m, v.to_register()); }, + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi8(p, m, v.to_register()); }, + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi16(p, m, v.to_register()); }, + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi32(p, m, v.to_register()); }, + [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, m, v.to_register()); }, - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi8(p, m, v.to_register()); }, - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi16(p, m, v.to_register()); }, - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi32(p, m, v.to_register()); }, - [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, m, v.to_register()); }, + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi8(p, m, v.to_register()); }, + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi16(p, m, v.to_register()); }, + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi32(p, m, v.to_register()); }, + [=](zmm_register auto v, auto m, auto p) { _mm512_mask_storeu_epi64(p, m, v.to_register()); }, [=](xmm_register auto v, auto m, auto p) { _mm_mask_storeu_epi64(p, dupBits(m), v.to_register()); }, [=](ymm_register auto v, auto m, auto p) { _mm256_mask_storeu_epi64(p, dupBits(m), v.to_register()); }, @@ -610,20 +610,20 @@ constexpr _V load_masked(x86_avx512_tag, std::span<_Up, _Extent> from, const typ // different to the type of the output vec, and therefore be unrepresentable // during the load. target_overloads call_insn { - [=](xmm_register auto, auto m, auto p) { return _mm_maskz_loadu_epi8(m, p); }, - [=](xmm_register auto, auto m, auto p) { return _mm_maskz_loadu_epi16(m, p); }, - [=](xmm_register auto, auto m, auto p) { return _mm_maskz_loadu_epi32(m, p); }, - [=](xmm_register auto, auto m, auto p) { return _mm_maskz_loadu_epi64(m, p); }, + [=](xmm_register auto, auto m, auto p) { return _mm_maskz_loadu_epi8(m, p); }, + [=](xmm_register auto, auto m, auto p) { return _mm_maskz_loadu_epi16(m, p); }, + [=](xmm_register auto, auto m, auto p) { return _mm_maskz_loadu_epi32(m, p); }, + [=](xmm_register auto, auto m, auto p) { return _mm_maskz_loadu_epi64(m, p); }, - [=](ymm_register auto, auto m, auto p) { return _mm256_maskz_loadu_epi8(m, p); }, - [=](ymm_register auto, auto m, auto p) { return _mm256_maskz_loadu_epi16(m, p); }, - [=](ymm_register auto, auto m, auto p) { return _mm256_maskz_loadu_epi32(m, p); }, - [=](ymm_register auto, auto m, auto p) { return _mm256_maskz_loadu_epi64(m, p); }, + [=](ymm_register auto, auto m, auto p) { return _mm256_maskz_loadu_epi8(m, p); }, + [=](ymm_register auto, auto m, auto p) { return _mm256_maskz_loadu_epi16(m, p); }, + [=](ymm_register auto, auto m, auto p) { return _mm256_maskz_loadu_epi32(m, p); }, + [=](ymm_register auto, auto m, auto p) { return _mm256_maskz_loadu_epi64(m, p); }, - [=](zmm_register auto, auto m, auto p) { return _mm512_maskz_loadu_epi8(m, p); }, - [=](zmm_register auto, auto m, auto p) { return _mm512_maskz_loadu_epi16(m, p); }, - [=](zmm_register auto, auto m, auto p) { return _mm512_maskz_loadu_epi32(m, p); }, - [=](zmm_register auto, auto m, auto p) { return _mm512_maskz_loadu_epi64(m, p); }, + [=](zmm_register auto, auto m, auto p) { return _mm512_maskz_loadu_epi8(m, p); }, + [=](zmm_register auto, auto m, auto p) { return _mm512_maskz_loadu_epi16(m, p); }, + [=](zmm_register auto, auto m, auto p) { return _mm512_maskz_loadu_epi32(m, p); }, + [=](zmm_register auto, auto m, auto p) { return _mm512_maskz_loadu_epi64(m, p); }, //:TODO: DUP all mask bits in a single operation, and only then extract? Cheaper than dupping all bits in small units. [=](xmm_register auto, auto m, auto p) { return _mm_maskz_loadu_epi64(dupBits(m), p); }, @@ -694,29 +694,29 @@ constexpr auto gather_from(x86_avx2_tag, const _Range& r, #if defined(__AVX512F__) // 32-bit Intel AVX-512 - [=](xmm_register auto i, auto m) { return _mm_mmask_i32gather_epi32(__m128i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, - [=](ymm_register auto i, auto m) { return _mm256_mmask_i32gather_epi32(__m256i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, + [=](xmm_register auto i, auto m) { return _mm_mmask_i32gather_epi32(__m128i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, + [=](ymm_register auto i, auto m) { return _mm256_mmask_i32gather_epi32(__m256i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, - [=](xmm_register auto i, auto m) { return _mm_mmask_i64gather_epi64(__m128i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, - [=](ymm_register auto i, auto m) { return _mm256_mmask_i64gather_epi64(__m256i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, + [=](xmm_register auto i, auto m) { return _mm_mmask_i64gather_epi64(__m128i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, + [=](ymm_register auto i, auto m) { return _mm256_mmask_i64gather_epi64(__m256i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, - [=](zmm_register auto i, auto m) { return _mm512_mask_i32gather_epi32(__m512i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, - [=](zmm_register auto i, auto m) { return _mm512_mask_i64gather_epi64(__m512(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, + [=](zmm_register auto i, auto m) { return _mm512_mask_i32gather_epi32(__m512i(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, + [=](zmm_register auto i, auto m) { return _mm512_mask_i64gather_epi64(__m512(), m.to_register(), i.to_register(), data_ptr, sizeof(_Tp)); }, #elif defined(__AVX2__) // Note the explicit growth of the mask to the correct register size to // avoid inadvertently leaving uninitialised elements active. // 32-bit Intel AVX2 - [=](xmm_register auto i, auto m) + [=](xmm_register auto i, auto m) { return _mm_mask_i32gather_epi32(__m128i(), data_ptr, i.to_register(), grow<4>(m).to_register(), sizeof(_Tp)); }, - [=](ymm_register auto i, auto m) + [=](ymm_register auto i, auto m) { return _mm256_mask_i32gather_epi32(__m256i(), data_ptr, i.to_register(), grow<8>(m).to_register(), sizeof(_Tp)); }, // 64-bit Intel AVX2 - [=](xmm_register auto i, auto m) + [=](xmm_register auto i, auto m) { return _mm_mask_i64gather_epi64(__m128i(), data_ptr, i.to_register(), grow<2>(m).to_register(), sizeof(_Tp)); }, - [=](ymm_register auto i, auto m) + [=](ymm_register auto i, auto m) { return _mm256_mask_i64gather_epi64(__m256i(), data_ptr, i.to_register(), grow<4>(m).to_register(), sizeof(_Tp)); }, #endif @@ -726,7 +726,7 @@ constexpr auto gather_from(x86_avx2_tag, const _Range& r, // The indexes and the values to gether must be the same size, and at least // 32-bit signed to match the hardware expectation. - constexpr auto idxNumBytes = std::max({sizeof(int32_t), sizeof(_Tp), sizeof(_Idx)}); + constexpr auto idxNumBytes = std::max({sizeof(std::int32_t), sizeof(_Tp), sizeof(_Idx)}); using _IW = std::make_signed_t>; const auto rmax = std::min<_IW>(r.size(), std::numeric_limits<_Idx>::max()); @@ -772,10 +772,10 @@ constexpr auto gather_from(x86_avx2_tag, const _Range& r, inline auto scatterImpl = [](basic_vec<_Tp, _VT> v, - basic_vec idx, auto ptr, uint64_t m) + basic_vec idx, auto ptr, std::uint64_t m) { - constexpr uint64_t justMaskBits = (1ULL << basic_vec<_Tp, _VT>::size()) - 1; - const uint64_t nm = justMaskBits & m; + constexpr std::uint64_t justMaskBits = (1ULL << basic_vec<_Tp, _VT>::size()) - 1; + const std::uint64_t nm = justMaskBits & m; constexpr bool isTp32 = (sizeof(_Tp) == 4); @@ -805,7 +805,7 @@ scatter_to(x86_avx512_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r, const basic_vec<_Idx, _IdxAbi>& indexes, flags<_Flags...> flags) { auto ptr = std::to_address(r.begin()); - const auto rmax = int32_t(std::min(r.size(), size_t(std::numeric_limits<_Idx>::max()))); + const auto rmax = std::int32_t(std::min(r.size(), std::size_t(std::numeric_limits<_Idx>::max()))); // Bind extra pointer and default mask parameters to the scatter lambda. Note // that a range check is added if necessary too. @@ -821,7 +821,7 @@ scatter_to(x86_avx512_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r, // All scatter instructions can be made to use 32-bit indexes. // :TODO: A special case might be 64-bit values using 64-bit indexes - auto indexAs32 = vec::size>(indexes); + auto indexAs32 = vec::size>(indexes); chunked_invoke<(int)basic_vec<_Tp>::size>(impl, vec_as_container(values), indexAs32); } @@ -836,7 +836,7 @@ scatter_to(x86_avx512_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r, auto ptr = std::to_address(r.begin()); // Range check if required. - const auto rmax = _Idx(std::min(r.size(), size_t(std::numeric_limits<_Idx>::max()))); + const auto rmax = _Idx(std::min(r.size(), std::size_t(std::numeric_limits<_Idx>::max()))); auto rangeCheckedMask = contains_flag(flags) ? mask : (indexes >= _Idx(0) && indexes < rmax) && mask; @@ -848,7 +848,7 @@ scatter_to(x86_avx512_tag, const basic_vec<_Tp, _TpAbi>& values, _Range&& r, // All scatter instructions can be made to use 32-bit indexes. // :TODO: A special case might be 64-bit values using 64-bit indexes - auto indexAs32 = vec::size>(indexes); + auto indexAs32 = vec::size>(indexes); chunked_invoke<(int)basic_vec<_Tp>::size()>(impl, vec_as_container(values), indexAs32, rangeCheckedMask); } #endif @@ -1110,64 +1110,64 @@ saturating_cast(x86_avx512_tag, const basic_vec<_Tp, _Abi>& value) noexcept { target_overloads impl { // S16 -> S8 - [=](xmm_register auto in, int8_t) { return _mm_cvtsepi16_epi8(in.to_register()); }, - [=](ymm_register auto in, int8_t) { return _mm256_cvtsepi16_epi8(in.to_register()); }, - [=](zmm_register auto in, int8_t) { return _mm512_cvtsepi16_epi8(in.to_register()); }, + [=](xmm_register auto in, std::int8_t) { return _mm_cvtsepi16_epi8(in.to_register()); }, + [=](ymm_register auto in, std::int8_t) { return _mm256_cvtsepi16_epi8(in.to_register()); }, + [=](zmm_register auto in, std::int8_t) { return _mm512_cvtsepi16_epi8(in.to_register()); }, // U16 -> U8 - [=](xmm_register auto in, uint8_t) { return _mm_cvtusepi16_epi8(in.to_register()); }, - [=](ymm_register auto in, uint8_t) { return _mm256_cvtusepi16_epi8(in.to_register()); }, - [=](zmm_register auto in, uint8_t) { return _mm512_cvtusepi16_epi8(in.to_register()); }, + [=](xmm_register auto in, std::uint8_t) { return _mm_cvtusepi16_epi8(in.to_register()); }, + [=](ymm_register auto in, std::uint8_t) { return _mm256_cvtusepi16_epi8(in.to_register()); }, + [=](zmm_register auto in, std::uint8_t) { return _mm512_cvtusepi16_epi8(in.to_register()); }, // S32 -> S8 - [=](xmm_register auto in, int8_t) { return _mm_cvtsepi32_epi8(in.to_register()); }, - [=](ymm_register auto in, int8_t) { return _mm256_cvtsepi32_epi8(in.to_register()); }, - [=](zmm_register auto in, int8_t) { return _mm512_cvtsepi32_epi8(in.to_register()); }, + [=](xmm_register auto in, std::int8_t) { return _mm_cvtsepi32_epi8(in.to_register()); }, + [=](ymm_register auto in, std::int8_t) { return _mm256_cvtsepi32_epi8(in.to_register()); }, + [=](zmm_register auto in, std::int8_t) { return _mm512_cvtsepi32_epi8(in.to_register()); }, // U32 -> U8 - [=](xmm_register auto in, uint8_t) { return _mm_cvtusepi32_epi8(in.to_register()); }, - [=](ymm_register auto in, uint8_t) { return _mm256_cvtusepi32_epi8(in.to_register()); }, - [=](zmm_register auto in, uint8_t) { return _mm512_cvtusepi32_epi8(in.to_register()); }, + [=](xmm_register auto in, std::uint8_t) { return _mm_cvtusepi32_epi8(in.to_register()); }, + [=](ymm_register auto in, std::uint8_t) { return _mm256_cvtusepi32_epi8(in.to_register()); }, + [=](zmm_register auto in, std::uint8_t) { return _mm512_cvtusepi32_epi8(in.to_register()); }, // S32 -> S16 - [=](xmm_register auto in, int16_t) { return _mm_cvtsepi32_epi16(in.to_register()); }, - [=](ymm_register auto in, int16_t) { return _mm256_cvtsepi32_epi16(in.to_register()); }, - [=](zmm_register auto in, int16_t) { return _mm512_cvtsepi32_epi16(in.to_register()); }, + [=](xmm_register auto in, std::int16_t) { return _mm_cvtsepi32_epi16(in.to_register()); }, + [=](ymm_register auto in, std::int16_t) { return _mm256_cvtsepi32_epi16(in.to_register()); }, + [=](zmm_register auto in, std::int16_t) { return _mm512_cvtsepi32_epi16(in.to_register()); }, // U32 -> U16 - [=](xmm_register auto in, uint16_t) { return _mm_cvtusepi32_epi16(in.to_register()); }, - [=](ymm_register auto in, uint16_t) { return _mm256_cvtusepi32_epi16(in.to_register()); }, - [=](zmm_register auto in, uint16_t) { return _mm512_cvtusepi32_epi16(in.to_register()); }, + [=](xmm_register auto in, std::uint16_t) { return _mm_cvtusepi32_epi16(in.to_register()); }, + [=](ymm_register auto in, std::uint16_t) { return _mm256_cvtusepi32_epi16(in.to_register()); }, + [=](zmm_register auto in, std::uint16_t) { return _mm512_cvtusepi32_epi16(in.to_register()); }, // S64 -> S8 - [=](xmm_register auto in, int8_t) { return _mm_cvtsepi64_epi8(in.to_register()); }, - [=](ymm_register auto in, int8_t) { return _mm256_cvtsepi64_epi8(in.to_register()); }, - [=](zmm_register auto in, int8_t) { return _mm512_cvtsepi64_epi8(in.to_register()); }, + [=](xmm_register auto in, std::int8_t) { return _mm_cvtsepi64_epi8(in.to_register()); }, + [=](ymm_register auto in, std::int8_t) { return _mm256_cvtsepi64_epi8(in.to_register()); }, + [=](zmm_register auto in, std::int8_t) { return _mm512_cvtsepi64_epi8(in.to_register()); }, // U64 -> U8 - [=](xmm_register auto in, uint8_t) { return _mm_cvtusepi64_epi8(in.to_register()); }, - [=](ymm_register auto in, uint8_t) { return _mm256_cvtusepi64_epi8(in.to_register()); }, - [=](zmm_register auto in, uint8_t) { return _mm512_cvtusepi64_epi8(in.to_register()); }, + [=](xmm_register auto in, std::uint8_t) { return _mm_cvtusepi64_epi8(in.to_register()); }, + [=](ymm_register auto in, std::uint8_t) { return _mm256_cvtusepi64_epi8(in.to_register()); }, + [=](zmm_register auto in, std::uint8_t) { return _mm512_cvtusepi64_epi8(in.to_register()); }, // S64 -> S16 - [=](xmm_register auto in, int16_t) { return _mm_cvtsepi64_epi16(in.to_register()); }, - [=](ymm_register auto in, int16_t) { return _mm256_cvtsepi64_epi16(in.to_register()); }, - [=](zmm_register auto in, int16_t) { return _mm512_cvtsepi64_epi16(in.to_register()); }, + [=](xmm_register auto in, std::int16_t) { return _mm_cvtsepi64_epi16(in.to_register()); }, + [=](ymm_register auto in, std::int16_t) { return _mm256_cvtsepi64_epi16(in.to_register()); }, + [=](zmm_register auto in, std::int16_t) { return _mm512_cvtsepi64_epi16(in.to_register()); }, // U64 -> U16 - [=](xmm_register auto in, uint16_t) { return _mm_cvtusepi64_epi16(in.to_register()); }, - [=](ymm_register auto in, uint16_t) { return _mm256_cvtusepi64_epi16(in.to_register()); }, - [=](zmm_register auto in, uint16_t) { return _mm512_cvtusepi64_epi16(in.to_register()); }, + [=](xmm_register auto in, std::uint16_t) { return _mm_cvtusepi64_epi16(in.to_register()); }, + [=](ymm_register auto in, std::uint16_t) { return _mm256_cvtusepi64_epi16(in.to_register()); }, + [=](zmm_register auto in, std::uint16_t) { return _mm512_cvtusepi64_epi16(in.to_register()); }, // S64 -> S32 - [=](xmm_register auto in, int32_t) { return _mm_cvtsepi64_epi32(in.to_register()); }, - [=](ymm_register auto in, int32_t) { return _mm256_cvtsepi64_epi32(in.to_register()); }, - [=](zmm_register auto in, int32_t) { return _mm512_cvtsepi64_epi32(in.to_register()); }, + [=](xmm_register auto in, std::int32_t) { return _mm_cvtsepi64_epi32(in.to_register()); }, + [=](ymm_register auto in, std::int32_t) { return _mm256_cvtsepi64_epi32(in.to_register()); }, + [=](zmm_register auto in, std::int32_t) { return _mm512_cvtsepi64_epi32(in.to_register()); }, // U64 -> U32 - [=](xmm_register auto in, uint32_t) { return _mm_cvtusepi64_epi32(in.to_register()); }, - [=](ymm_register auto in, uint32_t) { return _mm256_cvtusepi64_epi32(in.to_register()); }, - [=](zmm_register auto in, uint32_t) { return _mm512_cvtusepi64_epi32(in.to_register()); }, + [=](xmm_register auto in, std::uint32_t) { return _mm_cvtusepi64_epi32(in.to_register()); }, + [=](ymm_register auto in, std::uint32_t) { return _mm256_cvtusepi64_epi32(in.to_register()); }, + [=](zmm_register auto in, std::uint32_t) { return _mm512_cvtusepi64_epi32(in.to_register()); }, [=](auto unhandled, auto out) { static_assert(dependent_false>, "No target overload"); return out;} @@ -1217,7 +1217,7 @@ shuffle_epi8_lut(_Gen, basic_vec indexes) auto rawIndexes = indexes.to_register(); // Build a LUT which contains as many lanes as the indexes. - constexpr auto lut = vec([](auto idx) { return _Gen{}(idx & 0xF); }); + constexpr auto lut = vec([](auto idx) { return _Gen{}(idx & 0xF); }); // Use the fast in-lane byte shuffle. return basic_vec(shuffle_epi8(lut.to_register(), rawIndexes)); @@ -1317,12 +1317,12 @@ constexpr basic_vec<_Tp, _Abi> clz(x86_sse_tag, const basic_vec<_Tp, _Abi>& x) n // been added to the result of the clz but by putting it on the values to // lookup the compiler can hoist it. const auto lowClz = - shuffle_epi8_lut([](auto idx) -> std::uint8_t { return std::countl_zero(std::uint8_t(idx & 0xF)); }, bytes & uint8_t(0xF)); + shuffle_epi8_lut([](auto idx) -> std::uint8_t { return std::countl_zero(std::uint8_t(idx & 0xF)); }, bytes & std::uint8_t(0xF)); const auto highClz = - shuffle_epi8_lut([](auto idx) -> std::uint8_t { return std::countl_zero(std::uint8_t(idx & 0xF)) - 4; }, bytes >> uint8_t(4)); + shuffle_epi8_lut([](auto idx) -> std::uint8_t { return std::countl_zero(std::uint8_t(idx & 0xF)) - 4; }, bytes >> std::uint8_t(4)); // Select which clz actually applies here. - return simd_bit_cast<_Tp>(select(bytes > uint8_t(15), highClz, lowClz)); + return simd_bit_cast<_Tp>(select(bytes > std::uint8_t(15), highClz, lowClz)); }; constexpr auto numElementBytes = sizeof(_Tp); @@ -1437,7 +1437,7 @@ constexpr basic_vec<_Tp, _Abi> bitreverse(x86_sse_tag, const basic_vec<_Tp, _Abi template requires (sizeof(typename _Vp::value_type) == 1) // Multi-byte reversal handled generically. constexpr _Vp bitreverse(x86_avxsnc_tag, const _Vp& x) noexcept - { return x86::gf2p8affine(x, uint64_t(0x8040201008040201)); } + { return x86::gf2p8affine(x, std::uint64_t(0x8040201008040201)); } template requires (detail::is_floating_point_v<_Tp>) @@ -1591,25 +1591,25 @@ constexpr basic_vec<_Tp, _Abi> saturating_add(x86_tag, const basic_vec<_Tp, _Abi { target_overloads impl { // 128-bit - [=](xmm_register auto x, auto y) { return _mm_adds_epu8(x.to_register(), y.to_register()); }, - [=](xmm_register auto x, auto y) { return _mm_adds_epi8(x.to_register(), y.to_register()); }, - [=](xmm_register auto x, auto y) { return _mm_adds_epu16(x.to_register(), y.to_register()); }, - [=](xmm_register auto x, auto y) { return _mm_adds_epi16(x.to_register(), y.to_register()); }, + [=](xmm_register auto x, auto y) { return _mm_adds_epu8(x.to_register(), y.to_register()); }, + [=](xmm_register auto x, auto y) { return _mm_adds_epi8(x.to_register(), y.to_register()); }, + [=](xmm_register auto x, auto y) { return _mm_adds_epu16(x.to_register(), y.to_register()); }, + [=](xmm_register auto x, auto y) { return _mm_adds_epi16(x.to_register(), y.to_register()); }, // 256-bit (but only in Intel AVX2 - Intel AVX has to break into 128-bit pieces instead). #if defined(__AVX2__) - [=](ymm_register auto x, auto y) { return _mm256_adds_epu8(x.to_register(), y.to_register()); }, - [=](ymm_register auto x, auto y) { return _mm256_adds_epi8(x.to_register(), y.to_register()); }, - [=](ymm_register auto x, auto y) { return _mm256_adds_epu16(x.to_register(), y.to_register()); }, - [=](ymm_register auto x, auto y) { return _mm256_adds_epi16(x.to_register(), y.to_register()); }, + [=](ymm_register auto x, auto y) { return _mm256_adds_epu8(x.to_register(), y.to_register()); }, + [=](ymm_register auto x, auto y) { return _mm256_adds_epi8(x.to_register(), y.to_register()); }, + [=](ymm_register auto x, auto y) { return _mm256_adds_epu16(x.to_register(), y.to_register()); }, + [=](ymm_register auto x, auto y) { return _mm256_adds_epi16(x.to_register(), y.to_register()); }, #endif // 512-bit #if defined(__AVX512F__) - [=](zmm_register auto x, auto y) { return _mm512_adds_epu8(x.to_register(), y.to_register()); }, - [=](zmm_register auto x, auto y) { return _mm512_adds_epi8(x.to_register(), y.to_register()); }, - [=](zmm_register auto x, auto y) { return _mm512_adds_epu16(x.to_register(), y.to_register()); }, - [=](zmm_register auto x, auto y) { return _mm512_adds_epi16(x.to_register(), y.to_register()); }, + [=](zmm_register auto x, auto y) { return _mm512_adds_epu8(x.to_register(), y.to_register()); }, + [=](zmm_register auto x, auto y) { return _mm512_adds_epi8(x.to_register(), y.to_register()); }, + [=](zmm_register auto x, auto y) { return _mm512_adds_epu16(x.to_register(), y.to_register()); }, + [=](zmm_register auto x, auto y) { return _mm512_adds_epi16(x.to_register(), y.to_register()); }, #endif [=](auto unhandled, auto) { static_assert(dependent_false, "No target intrinsics"); return unhandled;} @@ -1637,25 +1637,25 @@ constexpr basic_vec<_Tp, _Abi> saturating_sub(x86_tag, const basic_vec<_Tp, _Abi { target_overloads impl { // 128-bit - [=](xmm_register auto x, auto y) { return _mm_subs_epu8(x.to_register(), y.to_register()); }, - [=](xmm_register auto x, auto y) { return _mm_subs_epi8(x.to_register(), y.to_register()); }, - [=](xmm_register auto x, auto y) { return _mm_subs_epu16(x.to_register(), y.to_register()); }, - [=](xmm_register auto x, auto y) { return _mm_subs_epi16(x.to_register(), y.to_register()); }, + [=](xmm_register auto x, auto y) { return _mm_subs_epu8(x.to_register(), y.to_register()); }, + [=](xmm_register auto x, auto y) { return _mm_subs_epi8(x.to_register(), y.to_register()); }, + [=](xmm_register auto x, auto y) { return _mm_subs_epu16(x.to_register(), y.to_register()); }, + [=](xmm_register auto x, auto y) { return _mm_subs_epi16(x.to_register(), y.to_register()); }, // 256-bit (but only in Intel AVX2 - Intel AVX has to break into 128-bit pieces instead). #if defined(__AVX2__) - [=](ymm_register auto x, auto y) { return _mm256_subs_epu8(x.to_register(), y.to_register()); }, - [=](ymm_register auto x, auto y) { return _mm256_subs_epi8(x.to_register(), y.to_register()); }, - [=](ymm_register auto x, auto y) { return _mm256_subs_epu16(x.to_register(), y.to_register()); }, - [=](ymm_register auto x, auto y) { return _mm256_subs_epi16(x.to_register(), y.to_register()); }, + [=](ymm_register auto x, auto y) { return _mm256_subs_epu8(x.to_register(), y.to_register()); }, + [=](ymm_register auto x, auto y) { return _mm256_subs_epi8(x.to_register(), y.to_register()); }, + [=](ymm_register auto x, auto y) { return _mm256_subs_epu16(x.to_register(), y.to_register()); }, + [=](ymm_register auto x, auto y) { return _mm256_subs_epi16(x.to_register(), y.to_register()); }, #endif // 512-bit #if defined(__AVX512F__) - [=](zmm_register auto x, auto y) { return _mm512_subs_epu8(x.to_register(), y.to_register()); }, - [=](zmm_register auto x, auto y) { return _mm512_subs_epi8(x.to_register(), y.to_register()); }, - [=](zmm_register auto x, auto y) { return _mm512_subs_epu16(x.to_register(), y.to_register()); }, - [=](zmm_register auto x, auto y) { return _mm512_subs_epi16(x.to_register(), y.to_register()); }, + [=](zmm_register auto x, auto y) { return _mm512_subs_epu8(x.to_register(), y.to_register()); }, + [=](zmm_register auto x, auto y) { return _mm512_subs_epi8(x.to_register(), y.to_register()); }, + [=](zmm_register auto x, auto y) { return _mm512_subs_epu16(x.to_register(), y.to_register()); }, + [=](zmm_register auto x, auto y) { return _mm512_subs_epi16(x.to_register(), y.to_register()); }, #endif [=](auto unhandled, auto) { static_assert(dependent_false, "No target intrinsics"); return unhandled;}