From a83cb135b52e19efdc56154c2b1fc838aaf0bc14 Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 14 Aug 2026 07:35:38 +0000 Subject: [PATCH 1/4] Add LoongArch64 support --- README.md | 1 + docs/Doxyfile | 1 + include/xsimd/arch/xsimd_isa.hpp | 4 + include/xsimd/arch/xsimd_lsx.hpp | 573 ++++++++++++++++++ include/xsimd/config/xsimd_arch.hpp | 4 +- include/xsimd/config/xsimd_config.hpp | 35 +- include/xsimd/config/xsimd_cpu_features.hpp | 2 + .../config/xsimd_cpu_features_loongarch.hpp | 84 +++ include/xsimd/config/xsimd_cpuid.hpp | 5 + include/xsimd/types/xsimd_all_registers.hpp | 1 + include/xsimd/types/xsimd_lsx_register.hpp | 149 +++++ test/CMakeLists.txt | 2 +- test/test_arch.cpp | 6 +- test/test_batch_complex.cpp | 14 + test/test_cpu_features.cpp | 15 + 15 files changed, 890 insertions(+), 6 deletions(-) create mode 100644 include/xsimd/arch/xsimd_lsx.hpp create mode 100644 include/xsimd/config/xsimd_cpu_features_loongarch.hpp create mode 100644 include/xsimd/types/xsimd_lsx_register.hpp diff --git a/README.md b/README.md index 894cf098d..05894a6b4 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ ARM | NEON, NEON64, SVE128/256/512 (fixed vector size) WebAssembly | WASM powerpc64 | VSX RISC-V | RISC-V128/256/512 (fixed vector size) +LoongArch64 | LSX, LASX IBM Z (s390x)| VXE (IBM z14) ## Installation diff --git a/docs/Doxyfile b/docs/Doxyfile index 6ea1a5664..8ebf18f8a 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -24,6 +24,7 @@ INPUT = ../include/xsimd/types/xsimd_api.hpp \ ../include/xsimd/types/xsimd_neon64_register.hpp \ ../include/xsimd/types/xsimd_neon_register.hpp \ ../include/xsimd/types/xsimd_rvv_register.hpp \ + ../include/xsimd/types/xsimd_lsx_register.hpp \ ../include/xsimd/types/xsimd_sse2_register.hpp \ ../include/xsimd/types/xsimd_sse3_register.hpp \ ../include/xsimd/types/xsimd_sse4_1_register.hpp \ diff --git a/include/xsimd/arch/xsimd_isa.hpp b/include/xsimd/arch/xsimd_isa.hpp index 87aaa6c4a..22af9bf8f 100644 --- a/include/xsimd/arch/xsimd_isa.hpp +++ b/include/xsimd/arch/xsimd_isa.hpp @@ -154,6 +154,10 @@ #include "./xsimd_rvv.hpp" #endif +#if XSIMD_WITH_LSX || XSIMD_WITH_LASX +#include "./xsimd_lsx.hpp" +#endif + #if XSIMD_WITH_WASM #include "./xsimd_wasm.hpp" #endif diff --git a/include/xsimd/arch/xsimd_lsx.hpp b/include/xsimd/arch/xsimd_lsx.hpp new file mode 100644 index 000000000..60f0ba7d5 --- /dev/null +++ b/include/xsimd/arch/xsimd_lsx.hpp @@ -0,0 +1,573 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_LSX_HPP +#define XSIMD_LSX_HPP + +#include "../types/xsimd_batch_constant.hpp" +#include "../types/xsimd_lsx_register.hpp" + +#include +#include +#include +#include +#include +#include + +namespace xsimd +{ + namespace kernel + { + namespace detail + { + template + XSIMD_INLINE To loongarch_bit_cast(From const& value) noexcept + { + static_assert(sizeof(To) == sizeof(From), "incompatible vector sizes"); + To result; + __builtin_memcpy(&result, &value, sizeof(result)); + return result; + } + + template + using loongarch_unsigned_register_t = typename batch, A>::register_type; + + template + XSIMD_INLINE loongarch_unsigned_register_t loongarch_to_bits(batch const& value) noexcept + { + return loongarch_bit_cast>(value.data); + } + + template + XSIMD_INLINE typename batch::register_type loongarch_from_bits(loongarch_unsigned_register_t const& value) noexcept + { + return loongarch_bit_cast::register_type>(value); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + typename batch::register_type result {}; + for (std::size_t i = 0; i < batch::size; ++i) + { + result[i] = static_cast(self.data[i]); + } + return result; + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + typename batch::register_type result {}; + for (std::size_t i = 0; i < batch::size; ++i) + { + result[i] = static_cast(self.data[i]); + } + return result; + } + } + + // abs + template >> + XSIMD_INLINE batch abs(batch const& self, requires_arch) noexcept + { + typename batch::register_type result {}; + for (std::size_t i = 0; i < batch::size; ++i) + { + result[i] = std::abs(self.data[i]); + } + return result; + } + + // add + template >> + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + { + return self.data + other.data; + } + + // all + template + XSIMD_INLINE bool all(batch_bool const& self, requires_arch) noexcept + { + for (std::size_t i = 0; i < batch_bool::size; ++i) + { + if (self.data[i] == 0) + { + return false; + } + } + return true; + } + + // any + template + XSIMD_INLINE bool any(batch_bool const& self, requires_arch) noexcept + { + for (std::size_t i = 0; i < batch_bool::size; ++i) + { + if (self.data[i] != 0) + { + return true; + } + } + return false; + } + + // batch_bool_cast + template + XSIMD_INLINE batch_bool batch_bool_cast(batch_bool const& self, batch_bool const&, requires_arch) noexcept + { + using result_type = typename batch_bool::register_type; + return detail::loongarch_bit_cast(self.data); + } + + // bitwise operations + template + XSIMD_INLINE batch bitwise_and(batch const& self, batch const& other, requires_arch) noexcept + { + auto bits = detail::loongarch_to_bits(self) & detail::loongarch_to_bits(other); + return detail::loongarch_from_bits(bits); + } + + template + XSIMD_INLINE batch_bool bitwise_and(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return self.data & other.data; + } + + template + XSIMD_INLINE batch bitwise_andnot(batch const& self, batch const& other, requires_arch) noexcept + { + auto bits = detail::loongarch_to_bits(self) & ~detail::loongarch_to_bits(other); + return detail::loongarch_from_bits(bits); + } + + template + XSIMD_INLINE batch_bool bitwise_andnot(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return self.data & ~other.data; + } + + template + XSIMD_INLINE batch bitwise_not(batch const& self, requires_arch) noexcept + { + auto bits = ~detail::loongarch_to_bits(self); + return detail::loongarch_from_bits(bits); + } + + template + XSIMD_INLINE batch_bool bitwise_not(batch_bool const& self, requires_arch) noexcept + { + return ~self.data; + } + + template + XSIMD_INLINE batch bitwise_or(batch const& self, batch const& other, requires_arch) noexcept + { + auto bits = detail::loongarch_to_bits(self) | detail::loongarch_to_bits(other); + return detail::loongarch_from_bits(bits); + } + + template + XSIMD_INLINE batch_bool bitwise_or(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return self.data | other.data; + } + + template + XSIMD_INLINE batch bitwise_xor(batch const& self, batch const& other, requires_arch) noexcept + { + auto bits = detail::loongarch_to_bits(self) ^ detail::loongarch_to_bits(other); + return detail::loongarch_from_bits(bits); + } + + template + XSIMD_INLINE batch_bool bitwise_xor(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return self.data ^ other.data; + } + + template + XSIMD_INLINE batch bitwise_cast(batch const& self, batch const&, requires_arch) noexcept + { + using result_type = typename batch::register_type; + return detail::loongarch_bit_cast(self.data); + } + + template >> + XSIMD_INLINE batch bitwise_lshift(batch const& self, std::int32_t other, requires_arch) noexcept + { + return self.data << other; + } + + template >> + XSIMD_INLINE batch bitwise_rshift(batch const& self, std::int32_t other, requires_arch) noexcept + { + return self.data >> other; + } + + // div + template >> + XSIMD_INLINE batch div(batch const& self, batch const& other, requires_arch) noexcept + { + return self.data / other.data; + } + + // broadcast + template >> + XSIMD_INLINE batch broadcast(T value, requires_arch) noexcept + { + typename batch::register_type result {}; + for (std::size_t i = 0; i < batch::size; ++i) + { + result[i] = value; + } + return result; + } + + // comparisons + template + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept + { + using result_type = typename batch_bool::register_type; + return detail::loongarch_bit_cast(self.data == other.data); + } + + template + XSIMD_INLINE batch_bool eq(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return self.data == other.data; + } + + template + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + { + using result_type = typename batch_bool::register_type; + return detail::loongarch_bit_cast(self.data < other.data); + } + + template + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + using result_type = typename batch_bool::register_type; + return detail::loongarch_bit_cast(self.data <= other.data); + } + + template + XSIMD_INLINE batch_bool gt(batch const& self, batch const& other, requires_arch) noexcept + { + using result_type = typename batch_bool::register_type; + return detail::loongarch_bit_cast(self.data > other.data); + } + + template + XSIMD_INLINE batch_bool ge(batch const& self, batch const& other, requires_arch) noexcept + { + using result_type = typename batch_bool::register_type; + return detail::loongarch_bit_cast(self.data >= other.data); + } + + template + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + { + return bitwise_not(eq(self, other, loongarch {}), loongarch {}); + } + + template + XSIMD_INLINE batch_bool neq(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return self.data != other.data; + } + + // first + template >> + XSIMD_INLINE T first(batch const& self, requires_arch) noexcept + { + return self.data[0]; + } + + template + XSIMD_INLINE std::complex first(batch, A> const& self, requires_arch) noexcept + { + return { self.real().data[0], self.imag().data[0] }; + } + + // horizontal add of rows + template + XSIMD_INLINE batch haddp(batch const* row, requires_arch) noexcept + { + typename batch::register_type result {}; + for (std::size_t i = 0; i < batch::size; ++i) + { + T value = T(0); + for (std::size_t j = 0; j < batch::size; ++j) + { + value += row[i].data[j]; + } + result[i] = value; + } + return result; + } + + // load + template >> + XSIMD_INLINE batch load_unaligned(T const* mem, convert, requires_arch) noexcept + { + typename batch::register_type result; + __builtin_memcpy(&result, mem, sizeof(result)); + return result; + } + + template >> + XSIMD_INLINE batch load_aligned(T const* mem, convert, requires_arch) noexcept + { + return load_unaligned(mem, convert {}, loongarch {}); + } + + // load/store complex helpers + namespace detail + { + template + XSIMD_INLINE batch, A> load_complex(batch const& first_chunk, batch const& second_chunk, requires_arch) noexcept + { + constexpr std::size_t size = batch::size; + std::array real {}; + std::array imag {}; + for (std::size_t i = 0; i < size; ++i) + { + const std::size_t real_index = 2 * i; + const std::size_t imag_index = real_index + 1; + real[i] = real_index < size ? first_chunk.data[real_index] : second_chunk.data[real_index - size]; + imag[i] = imag_index < size ? first_chunk.data[imag_index] : second_chunk.data[imag_index - size]; + } + return { load_unaligned(real.data(), convert {}, loongarch {}), + load_unaligned(imag.data(), convert {}, loongarch {}) }; + } + + template + XSIMD_INLINE batch complex_low(batch, A> const& self, requires_arch) noexcept + { + constexpr std::size_t size = batch::size; + std::array result {}; + for (std::size_t i = 0; i < size; ++i) + { + const std::size_t source = i / 2; + result[i] = (i % 2 == 0) ? self.real().data[source] : self.imag().data[source]; + } + return load_unaligned(result.data(), convert {}, loongarch {}); + } + + template + XSIMD_INLINE batch complex_high(batch, A> const& self, requires_arch) noexcept + { + constexpr std::size_t size = batch::size; + std::array result {}; + for (std::size_t i = 0; i < size; ++i) + { + const std::size_t interleaved_index = size + i; + const std::size_t source = interleaved_index / 2; + result[i] = (interleaved_index % 2 == 0) ? self.real().data[source] : self.imag().data[source]; + } + return load_unaligned(result.data(), convert {}, loongarch {}); + } + } + + // max/min + template + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept + { + typename batch::register_type result {}; + for (std::size_t i = 0; i < batch::size; ++i) + { + result[i] = self.data[i] < other.data[i] ? other.data[i] : self.data[i]; + } + return result; + } + + template + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + typename batch::register_type result {}; + for (std::size_t i = 0; i < batch::size; ++i) + { + result[i] = self.data[i] < other.data[i] ? self.data[i] : other.data[i]; + } + return result; + } + + // mul/neg + template + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept + { + return self.data * other.data; + } + + template + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + return -self.data; + } + + // rsqrt + template >> + XSIMD_INLINE batch rsqrt(batch const& self, requires_arch) noexcept + { + typename batch::register_type result {}; + for (std::size_t i = 0; i < batch::size; ++i) + { + result[i] = T(1) / std::sqrt(self.data[i]); + } + return result; + } + + // isnan + template >> + XSIMD_INLINE batch_bool isnan(batch const& self, requires_arch) noexcept + { + return neq(self, self, loongarch {}); + } + + // select + template + XSIMD_INLINE batch select(batch_bool const& cond, batch const& true_br, batch const& false_br, requires_arch) noexcept + { + using bits_type = detail::loongarch_unsigned_register_t; + auto mask = detail::loongarch_bit_cast(cond.data); + auto true_bits = detail::loongarch_to_bits(true_br); + auto false_bits = detail::loongarch_to_bits(false_br); + return detail::loongarch_from_bits((mask & true_bits) | (~mask & false_bits)); + } + + template + XSIMD_INLINE batch select(batch_bool_constant const&, batch const& true_br, batch const& false_br, requires_arch) noexcept + { + return select(batch_bool { Values... }, true_br, false_br, loongarch {}); + } + + // set + template + XSIMD_INLINE batch set(batch const&, requires_arch, Values... values) noexcept + { + static_assert(sizeof...(Values) == batch::size, "consistent init"); + return typename batch::register_type { static_cast(values)... }; + } + + template + XSIMD_INLINE batch, A> set(batch, A> const&, requires_arch, Values... values) noexcept + { + return batch, A>(set(batch {}, loongarch {}, values.real()...), + set(batch {}, loongarch {}, values.imag()...)); + } + + template + XSIMD_INLINE batch_bool set(batch_bool const&, requires_arch, Values... values) noexcept + { + static_assert(sizeof...(Values) == batch_bool::size, "consistent init"); + using value_type = sized_uint_t; + return typename batch_bool::register_type { static_cast(values ? ~value_type(0) : value_type(0))... }; + } + + // sqrt + template >> + XSIMD_INLINE batch sqrt(batch const& self, requires_arch) noexcept + { + typename batch::register_type result {}; + for (std::size_t i = 0; i < batch::size; ++i) + { + result[i] = std::sqrt(self.data[i]); + } + return result; + } + + // byte slides + template + XSIMD_INLINE batch slide_left(batch const& self, requires_arch) noexcept + { + static_assert(N <= A::alignment(), "invalid byte slide"); + std::array input {}; + std::array output {}; + __builtin_memcpy(input.data(), &self.data, input.size()); + for (std::size_t i = N; i < output.size(); ++i) + { + output[i] = input[i - N]; + } + typename batch::register_type result; + __builtin_memcpy(&result, output.data(), output.size()); + return result; + } + + template + XSIMD_INLINE batch slide_right(batch const& self, requires_arch) noexcept + { + static_assert(N <= A::alignment(), "invalid byte slide"); + std::array input {}; + std::array output {}; + __builtin_memcpy(input.data(), &self.data, input.size()); + for (std::size_t i = 0; i + N < output.size(); ++i) + { + output[i] = input[i + N]; + } + typename batch::register_type result; + __builtin_memcpy(&result, output.data(), output.size()); + return result; + } + + // store + template >> + XSIMD_INLINE void store_unaligned(T* mem, batch const& self, requires_arch) noexcept + { + __builtin_memcpy(mem, &self.data, sizeof(self.data)); + } + + template >> + XSIMD_INLINE void store_aligned(T* mem, batch const& self, requires_arch) noexcept + { + store_unaligned(mem, self, loongarch {}); + } + + // sub + template + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + { + return self.data - other.data; + } + + // zip + template + XSIMD_INLINE batch zip_lo(batch const& self, batch const& other, requires_arch) noexcept + { + typename batch::register_type result {}; + constexpr std::size_t half = batch::size / 2; + for (std::size_t i = 0; i < half; ++i) + { + result[2 * i] = self.data[i]; + result[2 * i + 1] = other.data[i]; + } + return result; + } + + template + XSIMD_INLINE batch zip_hi(batch const& self, batch const& other, requires_arch) noexcept + { + typename batch::register_type result {}; + constexpr std::size_t half = batch::size / 2; + for (std::size_t i = 0; i < half; ++i) + { + result[2 * i] = self.data[i + half]; + result[2 * i + 1] = other.data[i + half]; + } + return result; + } + } +} + +#endif diff --git a/include/xsimd/config/xsimd_arch.hpp b/include/xsimd/config/xsimd_arch.hpp index ac5546bbc..a84d70d84 100644 --- a/include/xsimd/config/xsimd_arch.hpp +++ b/include/xsimd/config/xsimd_arch.hpp @@ -154,9 +154,10 @@ namespace xsimd using all_arm_architectures = typename detail::join, neon64, neon>>::type; using all_power_architectures = arch_list; using all_riscv_architectures = all_rvv_architectures; + using all_loongarch_architectures = arch_list; using all_wasm_architectures = arch_list; using all_s390x_architectures = arch_list; - using all_architectures = typename detail::join::type; + using all_architectures = typename detail::join::type; using supported_architectures = typename detail::supported::type; @@ -164,6 +165,7 @@ namespace xsimd using arm_arch = typename detail::supported::type::best; using power_arch = typename detail::supported::type::best; using riscv_arch = typename detail::supported::type::best; + using loongarch_arch = typename detail::supported::type::best; using s390x_arch = typename detail::supported::type::best; using best_arch = typename supported_architectures::best; diff --git a/include/xsimd/config/xsimd_config.hpp b/include/xsimd/config/xsimd_config.hpp index 4bd2a341b..be6fada32 100644 --- a/include/xsimd/config/xsimd_config.hpp +++ b/include/xsimd/config/xsimd_config.hpp @@ -512,6 +512,39 @@ #define XSIMD_RVV_BITS 0 #endif +/** + * @ingroup xsimd_config_macro + * + * Set to 1 if the target is the 64-bit LoongArch architecture family. + */ +#if defined(__loongarch64) || defined(__loongarch64__) || (defined(__loongarch_grlen) && __loongarch_grlen == 64) +#define XSIMD_TARGET_LOONGARCH64 1 +#else +#define XSIMD_TARGET_LOONGARCH64 0 +#endif + +/** + * @ingroup xsimd_config_macro + * + * Set to 1 if the 128-bit Loongson SIMD Extension is available. + */ +#if XSIMD_TARGET_LOONGARCH64 && (defined(__loongarch_sx) || defined(__loongarch_asx)) +#define XSIMD_WITH_LSX 1 +#else +#define XSIMD_WITH_LSX 0 +#endif + +/** + * @ingroup xsimd_config_macro + * + * Set to 1 if the 256-bit Loongson Advanced SIMD Extension is available. + */ +#if XSIMD_TARGET_LOONGARCH64 && defined(__loongarch_asx) +#define XSIMD_WITH_LASX 1 +#else +#define XSIMD_WITH_LASX 0 +#endif + /** * @ingroup xsimd_config_macro * @@ -626,7 +659,7 @@ #endif -#if !XSIMD_WITH_SSE2 && !XSIMD_WITH_SSE3 && !XSIMD_WITH_SSSE3 && !XSIMD_WITH_SSE4_1 && !XSIMD_WITH_SSE4_2 && !XSIMD_WITH_AVX && !XSIMD_WITH_AVX2 && !XSIMD_WITH_AVXVNNI && !XSIMD_WITH_FMA3_SSE && !XSIMD_WITH_FMA4 && !XSIMD_WITH_FMA3_AVX && !XSIMD_WITH_FMA3_AVX2 && !XSIMD_WITH_AVX512F && !XSIMD_WITH_AVX512CD && !XSIMD_WITH_AVX512VL && !XSIMD_WITH_AVX512DQ && !XSIMD_WITH_AVX512BW && !XSIMD_WITH_AVX512ER && !XSIMD_WITH_AVX512PF && !XSIMD_WITH_AVX512IFMA && !XSIMD_WITH_AVX512VBMI && !XSIMD_WITH_AVX512VBMI2 && !XSIMD_WITH_NEON && !XSIMD_WITH_NEON64 && !XSIMD_WITH_SVE && !XSIMD_WITH_RVV && !XSIMD_WITH_WASM && !XSIMD_WITH_VSX && !XSIMD_WITH_EMULATED && !XSIMD_WITH_VXE +#if !XSIMD_WITH_SSE2 && !XSIMD_WITH_SSE3 && !XSIMD_WITH_SSSE3 && !XSIMD_WITH_SSE4_1 && !XSIMD_WITH_SSE4_2 && !XSIMD_WITH_AVX && !XSIMD_WITH_AVX2 && !XSIMD_WITH_AVXVNNI && !XSIMD_WITH_FMA3_SSE && !XSIMD_WITH_FMA4 && !XSIMD_WITH_FMA3_AVX && !XSIMD_WITH_FMA3_AVX2 && !XSIMD_WITH_AVX512F && !XSIMD_WITH_AVX512CD && !XSIMD_WITH_AVX512VL && !XSIMD_WITH_AVX512DQ && !XSIMD_WITH_AVX512BW && !XSIMD_WITH_AVX512ER && !XSIMD_WITH_AVX512PF && !XSIMD_WITH_AVX512IFMA && !XSIMD_WITH_AVX512VBMI && !XSIMD_WITH_AVX512VBMI2 && !XSIMD_WITH_NEON && !XSIMD_WITH_NEON64 && !XSIMD_WITH_SVE && !XSIMD_WITH_RVV && !XSIMD_WITH_LSX && !XSIMD_WITH_LASX && !XSIMD_WITH_WASM && !XSIMD_WITH_VSX && !XSIMD_WITH_EMULATED && !XSIMD_WITH_VXE #define XSIMD_NO_SUPPORTED_ARCHITECTURE #endif diff --git a/include/xsimd/config/xsimd_cpu_features.hpp b/include/xsimd/config/xsimd_cpu_features.hpp index 5dcc00416..f6035223d 100644 --- a/include/xsimd/config/xsimd_cpu_features.hpp +++ b/include/xsimd/config/xsimd_cpu_features.hpp @@ -15,6 +15,7 @@ #include "./xsimd_cpu_features_arm.hpp" #include "./xsimd_cpu_features_ppc.hpp" #include "./xsimd_cpu_features_riscv.hpp" +#include "./xsimd_cpu_features_loongarch.hpp" #include "./xsimd_cpu_features_s390x.hpp" #include "./xsimd_cpu_features_x86.hpp" @@ -40,6 +41,7 @@ namespace xsimd class cpu_features : public s390x_cpu_features, public ppc_cpu_features, public riscv_cpu_features, + public loongarch_cpu_features, public arm_cpu_features, public x86_cpu_features { diff --git a/include/xsimd/config/xsimd_cpu_features_loongarch.hpp b/include/xsimd/config/xsimd_cpu_features_loongarch.hpp new file mode 100644 index 000000000..9ef616177 --- /dev/null +++ b/include/xsimd/config/xsimd_cpu_features_loongarch.hpp @@ -0,0 +1,84 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_CPU_FEATURES_LOONGARCH_HPP +#define XSIMD_CPU_FEATURES_LOONGARCH_HPP + +#include "./xsimd_config.hpp" +#include "./xsimd_getauxval.hpp" + +#if XSIMD_TARGET_LOONGARCH64 && XSIMD_HAVE_LINUX_GETAUXVAL +// HWCAP masks are architecture-specific and asm/hwcap.h is not available +// for every target. +#include +#endif + +namespace xsimd +{ + /** + * An opinionated CPU feature detection utility for LoongArch. + * + * On Linux, runtime detection uses getauxval to query the auxiliary vector. + * On other platforms, only compile-time information is used. + * + * This is well defined on all architectures. + * It will always return false on non-LoongArch architectures. + */ + class loongarch_cpu_features : private linux_hwcap_backend_default + { + public: + inline bool lsx() const noexcept; + inline bool lasx() const noexcept; + }; + + /******************** + * Implementation * + ********************/ + + inline bool loongarch_cpu_features::lsx() const noexcept + { +#if XSIMD_TARGET_LOONGARCH64 && XSIMD_HAVE_LINUX_GETAUXVAL +#ifdef HWCAP_LOONGARCH_LSX + constexpr unsigned long loongarch_hwcap_lsx = HWCAP_LOONGARCH_LSX; +#else + // Possibly missing on older Linux distributions + constexpr unsigned long loongarch_hwcap_lsx = 1ul << 4; +#endif +#ifdef HWCAP_LOONGARCH_LASX + constexpr unsigned long loongarch_hwcap_lasx = HWCAP_LOONGARCH_LASX; +#else + // Possibly missing on older Linux distributions + constexpr unsigned long loongarch_hwcap_lasx = 1ul << 5; +#endif + return hwcap().has_feature(loongarch_hwcap_lsx) + || hwcap().has_feature(loongarch_hwcap_lasx); +#else + return XSIMD_WITH_LSX || XSIMD_WITH_LASX; +#endif + } + + inline bool loongarch_cpu_features::lasx() const noexcept + { +#if XSIMD_TARGET_LOONGARCH64 && XSIMD_HAVE_LINUX_GETAUXVAL +#ifdef HWCAP_LOONGARCH_LASX + constexpr unsigned long loongarch_hwcap_lasx = HWCAP_LOONGARCH_LASX; +#else + // Possibly missing on older Linux distributions + constexpr unsigned long loongarch_hwcap_lasx = 1ul << 5; +#endif + return hwcap().has_feature(loongarch_hwcap_lasx); +#else + return XSIMD_WITH_LASX; +#endif + } +} + +#endif diff --git a/include/xsimd/config/xsimd_cpuid.hpp b/include/xsimd/config/xsimd_cpuid.hpp index 7466cd5f8..a82816c73 100644 --- a/include/xsimd/config/xsimd_cpuid.hpp +++ b/include/xsimd/config/xsimd_cpuid.hpp @@ -68,6 +68,8 @@ namespace xsimd ARCH_FIELD_EX(detail::rvv<512>, rvv512) ARCH_FIELD_EX(detail::rvv<256>, rvv256) ARCH_FIELD_EX(detail::rvv<128>, rvv128) + ARCH_FIELD(lsx) + ARCH_FIELD(lasx) ARCH_FIELD(wasm) ARCH_FIELD(vsx) ARCH_FIELD(vxe) @@ -90,6 +92,9 @@ namespace xsimd rvv256 = cpu.rvv() && (cpu.rvv_size_bytes() >= (256 / 8)); rvv512 = cpu.rvv() && (cpu.rvv_size_bytes() >= (512 / 8)); + lsx = cpu.lsx(); + lasx = cpu.lasx(); + neon = cpu.neon(); neon64 = cpu.neon64(); i8mm_neon64 = cpu.neon64() && cpu.i8mm(); diff --git a/include/xsimd/types/xsimd_all_registers.hpp b/include/xsimd/types/xsimd_all_registers.hpp index 75d771752..8bd27aabf 100644 --- a/include/xsimd/types/xsimd_all_registers.hpp +++ b/include/xsimd/types/xsimd_all_registers.hpp @@ -34,6 +34,7 @@ #include "./xsimd_neon64_register.hpp" #include "./xsimd_neon_register.hpp" #include "./xsimd_rvv_register.hpp" +#include "./xsimd_lsx_register.hpp" #include "./xsimd_sse2_register.hpp" #include "./xsimd_sse3_register.hpp" #include "./xsimd_sse4_1_register.hpp" diff --git a/include/xsimd/types/xsimd_lsx_register.hpp b/include/xsimd/types/xsimd_lsx_register.hpp new file mode 100644 index 000000000..5fdd1e64b --- /dev/null +++ b/include/xsimd/types/xsimd_lsx_register.hpp @@ -0,0 +1,149 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_LSX_REGISTER_HPP +#define XSIMD_LSX_REGISTER_HPP + +#include "../config/xsimd_config.hpp" +#include "../utils/xsimd_type_traits.hpp" +#include "./xsimd_common_arch.hpp" +#include "./xsimd_register.hpp" + +#include + +namespace xsimd +{ + struct loongarch : common + { + }; + + /** + * @ingroup architectures + * + * LoongArch 128-bit SIMD extension. + */ + struct lsx : loongarch + { + static constexpr bool supported() noexcept { return XSIMD_WITH_LSX; } + static constexpr bool available() noexcept { return true; } + static constexpr bool requires_alignment() noexcept { return true; } + static constexpr std::size_t alignment() noexcept { return 16; } + static constexpr char const* name() noexcept { return "loongarch64+lsx"; } + }; + + /** + * @ingroup architectures + * + * LoongArch 256-bit advanced SIMD extension. + */ + struct lasx : loongarch + { + static constexpr bool supported() noexcept { return XSIMD_WITH_LASX; } + static constexpr bool available() noexcept { return true; } + static constexpr bool requires_alignment() noexcept { return true; } + static constexpr std::size_t alignment() noexcept { return 32; } + static constexpr char const* name() noexcept { return "loongarch64+lasx"; } + }; + +#if XSIMD_WITH_LSX || XSIMD_WITH_LASX + namespace types + { + namespace detail + { + template + struct loongarch_vector_type; + +#define XSIMD_DECLARE_LOONGARCH_VECTOR_TYPE(T, BYTES) \ + template <> \ + struct loongarch_vector_type \ + { \ + typedef T type __attribute__((vector_size(BYTES))); \ + } + +#define XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(T) \ + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPE(T, 16); \ + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPE(T, 32) + + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(signed char); + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(unsigned char); + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(char); + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(short); + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(unsigned short); + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(int); + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(unsigned int); + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(long); + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(unsigned long); + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(long long); + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(unsigned long long); + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(float); + XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(double); + +#undef XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES +#undef XSIMD_DECLARE_LOONGARCH_VECTOR_TYPE + + template + using loongarch_vector_type_t = typename loongarch_vector_type::type; + + template + using lsx_vector_type_t = loongarch_vector_type_t; + + template + using lasx_vector_type_t = loongarch_vector_type_t; + } + +#define XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(T, ARCH) \ + XSIMD_DECLARE_SIMD_REGISTER(T, ARCH, detail::ARCH##_vector_type_t) + +#define XSIMD_DECLARE_LOONGARCH_SIMD_REGISTERS(ARCH) \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(signed char, ARCH); \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(unsigned char, ARCH); \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(char, ARCH); \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(short, ARCH); \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(unsigned short, ARCH); \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(int, ARCH); \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(unsigned int, ARCH); \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(long, ARCH); \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(unsigned long, ARCH); \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(long long, ARCH); \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(unsigned long long, ARCH); \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(float, ARCH); \ + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(double, ARCH) + +#if XSIMD_WITH_LSX + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTERS(lsx); +#endif +#if XSIMD_WITH_LASX + XSIMD_DECLARE_LOONGARCH_SIMD_REGISTERS(lasx); +#endif + +#undef XSIMD_DECLARE_LOONGARCH_SIMD_REGISTERS +#undef XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER + +#if XSIMD_WITH_LSX + template + struct get_bool_simd_register + { + using type = simd_register, lsx>; + }; +#endif + +#if XSIMD_WITH_LASX + template + struct get_bool_simd_register + { + using type = simd_register, lasx>; + }; +#endif + } +#endif +} + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a34dafde9..e9826a9b4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -97,7 +97,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU" message(STATUS "CMAKE_CXX_LINK_EXECUTABLE: ${CMAKE_CXX_LINK_EXECUTABLE}") elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "^ppc64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${TARGET_ARCH} -mtune=${TARGET_ARCH}") - elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64") + elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64|loongarch64") # Nothing specific elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "powerpc") # Nothing specific diff --git a/test/test_arch.cpp b/test/test_arch.cpp index 3f460bf27..d554930b9 100644 --- a/test/test_arch.cpp +++ b/test/test_arch.cpp @@ -160,7 +160,7 @@ TEST_CASE("[multi arch support]") using batch8i32 = xsimd::make_sized_batch_t; using batch8u32 = xsimd::make_sized_batch_t; -#if XSIMD_WITH_SSE2 || XSIMD_WITH_NEON || XSIMD_WITH_NEON64 || XSIMD_WITH_SVE || (XSIMD_WITH_RVV && XSIMD_RVV_BITS == 128) +#if XSIMD_WITH_SSE2 || XSIMD_WITH_NEON || XSIMD_WITH_NEON64 || XSIMD_WITH_SVE || XSIMD_WITH_LSX || (XSIMD_WITH_RVV && XSIMD_RVV_BITS == 128) CHECK_EQ(4, size_t(batch4f::size)); CHECK_EQ(4, size_t(batch4c::size)); CHECK_EQ(4, size_t(batch4i32::size)); @@ -171,7 +171,7 @@ TEST_CASE("[multi arch support]") CHECK_UNARY(bool(std::is_same_v)); CHECK_UNARY(bool(std::is_same_v)); -#if XSIMD_WITH_SSE2 || XSIMD_WITH_NEON64 || XSIMD_WITH_SVE || XSIMD_WITH_RVV +#if XSIMD_WITH_SSE2 || XSIMD_WITH_NEON64 || XSIMD_WITH_SVE || XSIMD_WITH_RVV || XSIMD_WITH_LSX CHECK_EQ(2, size_t(batch2d::size)); CHECK_EQ(2, size_t(batch2z::size)); CHECK_UNARY(bool(std::is_same_v)); @@ -181,7 +181,7 @@ TEST_CASE("[multi arch support]") #endif #endif -#if !XSIMD_WITH_AVX && !XSIMD_WITH_FMA3 && !(XSIMD_WITH_SVE && XSIMD_SVE_BITS == 256) && !(XSIMD_WITH_RVV && XSIMD_RVV_BITS == 256) +#if !XSIMD_WITH_AVX && !XSIMD_WITH_FMA3 && !XSIMD_WITH_LASX && !(XSIMD_WITH_SVE && XSIMD_SVE_BITS == 256) && !(XSIMD_WITH_RVV && XSIMD_RVV_BITS == 256) CHECK_UNARY(bool(std::is_same_v)); CHECK_UNARY(bool(std::is_same_v)); CHECK_UNARY(bool(std::is_same_v)); diff --git a/test/test_batch_complex.cpp b/test/test_batch_complex.cpp index cccb4bee1..94bb799fd 100644 --- a/test/test_batch_complex.cpp +++ b/test/test_batch_complex.cpp @@ -17,6 +17,7 @@ #include #include #include +#include using namespace std::placeholders; @@ -199,6 +200,11 @@ struct batch_complex_test imag[i] = lhs[i].imag(); tmp[i] = value_type(real[i]); } + +#if XSIMD_WITH_LSX || XSIMD_WITH_LASX || XSIMD_WITH_SVE || XSIMD_WITH_RVV + batch_type b2 = make_batch(lhs, std::make_index_sequence {}); + CHECK_EQ(b2, lhs); +#endif } void test_access_operator() const @@ -704,6 +710,14 @@ struct batch_complex_test #endif private: +#if XSIMD_WITH_LSX || XSIMD_WITH_LASX || XSIMD_WITH_SVE || XSIMD_WITH_RVV + template + batch_type make_batch(array_type const& values, std::index_sequence) const + { + return batch_type(values[I]...); + } +#endif + batch_type batch_lhs() const { batch_type res = batch_type::load_unaligned(lhs.data()); diff --git a/test/test_cpu_features.cpp b/test/test_cpu_features.cpp index 05958a204..cf1c9ad33 100644 --- a/test/test_cpu_features.cpp +++ b/test/test_cpu_features.cpp @@ -172,6 +172,21 @@ TEST_CASE("[cpu_features] risc-v features from environment") CHECK_ENV_FEATURE("XSIMD_TEST_CPU_ASSUME_RVV", cpu.rvv()); } +TEST_CASE("[cpu_features] LoongArch implication chains") +{ + xsimd::cpu_features cpu; + + CHECK_IMPLICATION(cpu.lasx(), cpu.lsx()); +} + +TEST_CASE("[cpu_features] LoongArch features from environment") +{ + xsimd::cpu_features cpu; + + CHECK_ENV_FEATURE("XSIMD_TEST_CPU_ASSUME_LSX", cpu.lsx()); + CHECK_ENV_FEATURE("XSIMD_TEST_CPU_ASSUME_LASX", cpu.lasx()); +} + TEST_CASE("[cpu_features] ppc features from environment") { xsimd::cpu_features cpu; From d792486b597a866c0539167e730a1e3124fc6053 Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 17 Aug 2026 01:49:32 +0000 Subject: [PATCH 2/4] Add LoongArch64 cross-compilation CI --- .../gcc-loongarch64-linux-gnu.cmake | 4 ++ .github/workflows/cross-loongarch.yml | 65 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 .github/toolchains/gcc-loongarch64-linux-gnu.cmake create mode 100644 .github/workflows/cross-loongarch.yml diff --git a/.github/toolchains/gcc-loongarch64-linux-gnu.cmake b/.github/toolchains/gcc-loongarch64-linux-gnu.cmake new file mode 100644 index 000000000..472b4efb2 --- /dev/null +++ b/.github/toolchains/gcc-loongarch64-linux-gnu.cmake @@ -0,0 +1,4 @@ +set(CMAKE_SYSTEM_PROCESSOR loongarch64) +set(triple loongarch64-linux-gnu) + +include(${CMAKE_CURRENT_LIST_DIR}/gcc.cmake) diff --git a/.github/workflows/cross-loongarch.yml b/.github/workflows/cross-loongarch.yml new file mode 100644 index 000000000..6abbd4907 --- /dev/null +++ b/.github/workflows/cross-loongarch.yml @@ -0,0 +1,65 @@ +name: LoongArch64 cross-compilation build +on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.job }}-${{ github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + name: 'LoongArch64 ${{ matrix.simd.name }}' + strategy: + fail-fast: false + matrix: + simd: + - { name: 'LSX', flags: '-mlsx' } + - { name: 'LASX', flags: '-mlasx' } + steps: + - name: Setup compiler + run: | + sudo apt-get -y -qq update + sudo apt-get -y -qq --no-install-suggests --no-install-recommends install \ + gcc-14-loongarch64-linux-gnu \ + g++-14-loongarch64-linux-gnu + sudo update-alternatives --install /usr/bin/loongarch64-linux-gnu-gcc \ + loongarch64-linux-gnu-gcc /usr/bin/loongarch64-linux-gnu-gcc-14 20 + sudo update-alternatives --install /usr/bin/loongarch64-linux-gnu-g++ \ + loongarch64-linux-gnu-g++ /usr/bin/loongarch64-linux-gnu-g++-14 20 + loongarch64-linux-gnu-g++ --version + - name: Setup QEMU + run: | + sudo apt-get -y -qq update + sudo apt-get -y -qq --no-install-suggests --no-install-recommends install qemu-user-static + qemu-loongarch64-static --version + - name: Setup Ninja + run: | + sudo apt-get -y -qq install ninja-build + - name: Checkout xsimd + uses: actions/checkout@v6 + - name: Setup + run: >- + cmake -S . -B _build + -GNinja + -DBUILD_TESTS=ON + -DDOWNLOAD_DOCTEST=ON + -DCMAKE_BUILD_TYPE=Release + -DTARGET_ARCH=generic + -DCMAKE_C_FLAGS="${{ matrix.simd.flags }}" + -DCMAKE_CXX_FLAGS="${{ matrix.simd.flags }}" + -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/.github/toolchains/gcc-loongarch64-linux-gnu.cmake + - name: Build + run: cmake --build _build + - name: Set CPU feature test expectations + run: | + echo "XSIMD_TEST_CPU_ASSUME_SSE4_2=0" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_NEON64=0" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_SVE=0" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_RVV=0" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_LSX=1" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_LASX=1" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_VSX=0" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_VXE=0" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_MANUFACTURER=unknown" >> "$GITHUB_ENV" + - name: Testing xsimd + timeout-minutes: 15 + run: qemu-loongarch64-static -L /usr/loongarch64-linux-gnu ./test/test_xsimd + working-directory: ${{ github.workspace }}/_build From c2f102109bc8c03d712fc6d58e911cf9971d391c Mon Sep 17 00:00:00 2001 From: chris Date: Mon, 17 Aug 2026 09:35:45 +0000 Subject: [PATCH 3/4] Make lasx derive from lsx --- include/xsimd/types/xsimd_lsx_register.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xsimd/types/xsimd_lsx_register.hpp b/include/xsimd/types/xsimd_lsx_register.hpp index 5fdd1e64b..f8d2f66ce 100644 --- a/include/xsimd/types/xsimd_lsx_register.hpp +++ b/include/xsimd/types/xsimd_lsx_register.hpp @@ -44,7 +44,7 @@ namespace xsimd * * LoongArch 256-bit advanced SIMD extension. */ - struct lasx : loongarch + struct lasx : lsx { static constexpr bool supported() noexcept { return XSIMD_WITH_LASX; } static constexpr bool available() noexcept { return true; } From 57b062ea7fd336e253fa084a84827d37c3cb5efd Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 21 Aug 2026 02:07:27 +0000 Subject: [PATCH 4/4] Refactor LoongArch LSX and LASX backends --- .github/workflows/cross-loongarch.yml | 8 +- docs/Doxyfile | 1 + include/xsimd/arch/xsimd_isa.hpp | 6 +- include/xsimd/arch/xsimd_lasx.hpp | 1012 +++++++++++++++++ include/xsimd/arch/xsimd_lsx.hpp | 928 ++++++++++----- .../config/xsimd_cpu_features_loongarch.hpp | 10 +- include/xsimd/types/xsimd_all_registers.hpp | 1 + include/xsimd/types/xsimd_lasx_register.hpp | 67 ++ include/xsimd/types/xsimd_lsx_register.hpp | 121 +- test/test_batch_cast.cpp | 2 +- 10 files changed, 1753 insertions(+), 403 deletions(-) create mode 100644 include/xsimd/arch/xsimd_lasx.hpp create mode 100644 include/xsimd/types/xsimd_lasx_register.hpp diff --git a/.github/workflows/cross-loongarch.yml b/.github/workflows/cross-loongarch.yml index 6abbd4907..8840fdd86 100644 --- a/.github/workflows/cross-loongarch.yml +++ b/.github/workflows/cross-loongarch.yml @@ -11,8 +11,8 @@ jobs: fail-fast: false matrix: simd: - - { name: 'LSX', flags: '-mlsx' } - - { name: 'LASX', flags: '-mlasx' } + - { name: 'LSX', flags: '-mlsx -mno-lasx', lasx: 0, cpu: 'la464,lsx=on,lasx=off' } + - { name: 'LASX', flags: '-mlasx', lasx: 1, cpu: 'la464,lsx=on,lasx=on' } steps: - name: Setup compiler run: | @@ -55,11 +55,11 @@ jobs: echo "XSIMD_TEST_CPU_ASSUME_SVE=0" >> "$GITHUB_ENV" echo "XSIMD_TEST_CPU_ASSUME_RVV=0" >> "$GITHUB_ENV" echo "XSIMD_TEST_CPU_ASSUME_LSX=1" >> "$GITHUB_ENV" - echo "XSIMD_TEST_CPU_ASSUME_LASX=1" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_LASX=${{ matrix.simd.lasx }}" >> "$GITHUB_ENV" echo "XSIMD_TEST_CPU_ASSUME_VSX=0" >> "$GITHUB_ENV" echo "XSIMD_TEST_CPU_ASSUME_VXE=0" >> "$GITHUB_ENV" echo "XSIMD_TEST_CPU_ASSUME_MANUFACTURER=unknown" >> "$GITHUB_ENV" - name: Testing xsimd timeout-minutes: 15 - run: qemu-loongarch64-static -L /usr/loongarch64-linux-gnu ./test/test_xsimd + run: qemu-loongarch64-static -cpu '${{ matrix.simd.cpu }}' -L /usr/loongarch64-linux-gnu ./test/test_xsimd working-directory: ${{ github.workspace }}/_build diff --git a/docs/Doxyfile b/docs/Doxyfile index 8ebf18f8a..1c5e5b6db 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -24,6 +24,7 @@ INPUT = ../include/xsimd/types/xsimd_api.hpp \ ../include/xsimd/types/xsimd_neon64_register.hpp \ ../include/xsimd/types/xsimd_neon_register.hpp \ ../include/xsimd/types/xsimd_rvv_register.hpp \ + ../include/xsimd/types/xsimd_lasx_register.hpp \ ../include/xsimd/types/xsimd_lsx_register.hpp \ ../include/xsimd/types/xsimd_sse2_register.hpp \ ../include/xsimd/types/xsimd_sse3_register.hpp \ diff --git a/include/xsimd/arch/xsimd_isa.hpp b/include/xsimd/arch/xsimd_isa.hpp index 22af9bf8f..8077a8a49 100644 --- a/include/xsimd/arch/xsimd_isa.hpp +++ b/include/xsimd/arch/xsimd_isa.hpp @@ -154,10 +154,14 @@ #include "./xsimd_rvv.hpp" #endif -#if XSIMD_WITH_LSX || XSIMD_WITH_LASX +#if XSIMD_WITH_LSX #include "./xsimd_lsx.hpp" #endif +#if XSIMD_WITH_LASX +#include "./xsimd_lasx.hpp" +#endif + #if XSIMD_WITH_WASM #include "./xsimd_wasm.hpp" #endif diff --git a/include/xsimd/arch/xsimd_lasx.hpp b/include/xsimd/arch/xsimd_lasx.hpp new file mode 100644 index 000000000..e5c92a319 --- /dev/null +++ b/include/xsimd/arch/xsimd_lasx.hpp @@ -0,0 +1,1012 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_LASX_HPP +#define XSIMD_LASX_HPP + +#include "../types/xsimd_batch_constant.hpp" +#include "../types/xsimd_lasx_register.hpp" +#include "../types/xsimd_utils.hpp" + +#include +#include +#include +#include + +namespace xsimd +{ + namespace kernel + { + namespace detail + { + template + struct lasx_set_vector + { + typedef T type __attribute__((vector_size(32))); + }; + + template + XSIMD_INLINE __m256i lasx_to_int(batch const& value) noexcept + { + return bit_cast<__m256i>(value.data); + } + + template + XSIMD_INLINE typename batch::register_type lasx_from_int(__m256i value) noexcept + { + return bit_cast::register_type>(value); + } + + // LASX mask extraction is 128-bit lane-local. Reorder or merge + // both halves so bit i always represents logical lane i. + template + XSIMD_INLINE std::uint32_t lasx_mask(__m256i value) noexcept + { + if constexpr (sizeof(T) == 1) + { + const __m256i sign_bits = __lasx_xvmskltz_b(value); + const auto lo = static_cast(__lasx_xvpickve2gr_w(sign_bits, 0)); + const auto hi = static_cast(__lasx_xvpickve2gr_w(sign_bits, 4)); + return lo | (hi << 16); + } + else if constexpr (sizeof(T) == 2) + { + const __m256i odd = __lasx_xvpickod_b(value, value); + const __m256i shuffled = __lasx_xvpermi_d(odd, 0xd8); + return static_cast(__lasx_xvpickve2gr_w(__lasx_xvmskltz_b(shuffled), 0)); + } + else if constexpr (sizeof(T) == 4) + { + const __m256i odd = __lasx_xvpickod_h(value, value); + const __m256i shuffled = __lasx_xvpermi_d(odd, 0xd8); + return static_cast(__lasx_xvpickve2gr_w(__lasx_xvmskltz_h(shuffled), 0)); + } + else + { + const __m256i odd = __lasx_xvpickod_w(value, value); + const __m256i shuffled = __lasx_xvpermi_d(odd, 0xd8); + return static_cast(__lasx_xvpickve2gr_w(__lasx_xvmskltz_w(shuffled), 0)); + } + } + + XSIMD_INLINE __m256 lasx_hadd_pair(__m256 lhs, __m256 rhs) noexcept + { + const __m256i even = __lasx_xvpickev_w(bit_cast<__m256i>(rhs), bit_cast<__m256i>(lhs)); + const __m256i odd = __lasx_xvpickod_w(bit_cast<__m256i>(rhs), bit_cast<__m256i>(lhs)); + return __lasx_xvfadd_s(bit_cast<__m256>(even), bit_cast<__m256>(odd)); + } + + XSIMD_INLINE __m256d lasx_hadd_pair(__m256d lhs, __m256d rhs) noexcept + { + const __m256i even = __lasx_xvpickev_d(bit_cast<__m256i>(rhs), bit_cast<__m256i>(lhs)); + const __m256i odd = __lasx_xvpickod_d(bit_cast<__m256i>(rhs), bit_cast<__m256i>(lhs)); + return __lasx_xvfadd_d(bit_cast<__m256d>(even), bit_cast<__m256d>(odd)); + } + + // fast_cast + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvffint_s_w(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvffint_s_wu(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvffint_d_l(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvffint_d_lu(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvftintrz_w_s(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvftintrz_wu_s(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvftintrz_l_d(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvftintrz_lu_d(self.data); + } + } + + // abs + template + XSIMD_INLINE batch abs(batch const& self, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvbitclri_w(detail::lasx_to_int(self), 31)); + } + + template + XSIMD_INLINE batch abs(batch const& self, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvbitclri_d(detail::lasx_to_int(self), 63)); + } + + // add + template >> + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvadd_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvadd_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvadd_w(self.data, other.data); + else + return __lasx_xvadd_d(self.data, other.data); + } + + template + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfadd_s(self.data, other.data); + } + + template + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfadd_d(self.data, other.data); + } + + // all/any + template + XSIMD_INLINE bool all(batch_bool const& self, requires_arch) noexcept + { + constexpr std::uint64_t all_bits = (std::uint64_t(1) << batch_bool::size) - 1; + return detail::lasx_mask(self.data) == all_bits; + } + + template + XSIMD_INLINE bool any(batch_bool const& self, requires_arch) noexcept + { + return detail::lasx_mask(self.data) != 0; + } + + // batch_bool_cast + template + XSIMD_INLINE batch_bool batch_bool_cast(batch_bool const& self, batch_bool const&, requires_arch) noexcept + { + return self.data; + } + + // bitwise operations + template + XSIMD_INLINE batch bitwise_and(batch const& self, batch const& other, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvand_v(detail::lasx_to_int(self), detail::lasx_to_int(other))); + } + + template + XSIMD_INLINE batch_bool bitwise_and(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lasx_xvand_v(self.data, other.data); + } + + template + XSIMD_INLINE batch bitwise_andnot(batch const& self, batch const& other, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvandn_v(detail::lasx_to_int(other), detail::lasx_to_int(self))); + } + + template + XSIMD_INLINE batch_bool bitwise_andnot(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lasx_xvandn_v(other.data, self.data); + } + + template + XSIMD_INLINE batch bitwise_not(batch const& self, requires_arch) noexcept + { + const __m256i bits = detail::lasx_to_int(self); + return detail::lasx_from_int(__lasx_xvnor_v(bits, bits)); + } + + template + XSIMD_INLINE batch_bool bitwise_not(batch_bool const& self, requires_arch) noexcept + { + return __lasx_xvnor_v(self.data, self.data); + } + + template + XSIMD_INLINE batch bitwise_or(batch const& self, batch const& other, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvor_v(detail::lasx_to_int(self), detail::lasx_to_int(other))); + } + + template + XSIMD_INLINE batch_bool bitwise_or(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lasx_xvor_v(self.data, other.data); + } + + template + XSIMD_INLINE batch bitwise_xor(batch const& self, batch const& other, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvxor_v(detail::lasx_to_int(self), detail::lasx_to_int(other))); + } + + template + XSIMD_INLINE batch_bool bitwise_xor(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lasx_xvxor_v(self.data, other.data); + } + + template + XSIMD_INLINE batch bitwise_cast(batch const& self, batch const&, requires_arch) noexcept + { + return bit_cast::register_type>(self.data); + } + + // shifts + template >> + XSIMD_INLINE batch bitwise_lshift(batch const& self, std::int32_t other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvsll_b(self.data, __lasx_xvreplgr2vr_b(other)); + else if constexpr (sizeof(T) == 2) + return __lasx_xvsll_h(self.data, __lasx_xvreplgr2vr_h(other)); + else if constexpr (sizeof(T) == 4) + return __lasx_xvsll_w(self.data, __lasx_xvreplgr2vr_w(other)); + else + return __lasx_xvsll_d(self.data, __lasx_xvreplgr2vr_d(other)); + } + + template >> + XSIMD_INLINE batch bitwise_rshift(batch const& self, std::int32_t other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lasx_xvsra_b(self.data, __lasx_xvreplgr2vr_b(other)); + else if constexpr (sizeof(T) == 2) + return __lasx_xvsra_h(self.data, __lasx_xvreplgr2vr_h(other)); + else if constexpr (sizeof(T) == 4) + return __lasx_xvsra_w(self.data, __lasx_xvreplgr2vr_w(other)); + else + return __lasx_xvsra_d(self.data, __lasx_xvreplgr2vr_d(other)); + } + else + { + if constexpr (sizeof(T) == 1) + return __lasx_xvsrl_b(self.data, __lasx_xvreplgr2vr_b(other)); + else if constexpr (sizeof(T) == 2) + return __lasx_xvsrl_h(self.data, __lasx_xvreplgr2vr_h(other)); + else if constexpr (sizeof(T) == 4) + return __lasx_xvsrl_w(self.data, __lasx_xvreplgr2vr_w(other)); + else + return __lasx_xvsrl_d(self.data, __lasx_xvreplgr2vr_d(other)); + } + } + + // div + template + XSIMD_INLINE batch div(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfdiv_s(self.data, other.data); + } + + template + XSIMD_INLINE batch div(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfdiv_d(self.data, other.data); + } + + // broadcast + template >> + XSIMD_INLINE batch broadcast(T value, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvreplgr2vr_b(static_cast(value)); + else if constexpr (sizeof(T) == 2) + return __lasx_xvreplgr2vr_h(static_cast(value)); + else if constexpr (sizeof(T) == 4) + return __lasx_xvreplgr2vr_w(bit_cast>(value)); + else + return __lasx_xvreplgr2vr_d(bit_cast>(value)); + } + + template + XSIMD_INLINE batch broadcast(float value, requires_arch) noexcept + { + const auto bits = bit_cast(value); + return bit_cast<__m256>(__lasx_xvreplgr2vr_w(bits)); + } + + template + XSIMD_INLINE batch broadcast(double value, requires_arch) noexcept + { + const auto bits = bit_cast(value); + return bit_cast<__m256d>(__lasx_xvreplgr2vr_d(static_cast(bits))); + } + + // comparisons + template >> + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvseq_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvseq_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvseq_w(self.data, other.data); + else + return __lasx_xvseq_d(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_ceq_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_ceq_d(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool eq(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvseq_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvseq_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvseq_w(self.data, other.data); + else + return __lasx_xvseq_d(self.data, other.data); + } + + template >> + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lasx_xvslt_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvslt_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvslt_w(self.data, other.data); + else + return __lasx_xvslt_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lasx_xvslt_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvslt_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvslt_wu(self.data, other.data); + else + return __lasx_xvslt_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_clt_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_clt_d(self.data, other.data); + } + + template >> + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lasx_xvsle_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvsle_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvsle_w(self.data, other.data); + else + return __lasx_xvsle_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lasx_xvsle_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvsle_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvsle_wu(self.data, other.data); + else + return __lasx_xvsle_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_cle_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_cle_d(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool gt(batch const& self, batch const& other, requires_arch) noexcept + { + return lt(other, self, lasx {}); + } + + template + XSIMD_INLINE batch_bool ge(batch const& self, batch const& other, requires_arch) noexcept + { + return le(other, self, lasx {}); + } + + template >> + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + { + const __m256i equal = eq(self, other, lasx {}).data; + return __lasx_xvnor_v(equal, equal); + } + + template + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_cune_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_cune_d(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool neq(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lasx_xvxor_v(self.data, other.data); + } + + // first + template >> + XSIMD_INLINE T first(batch const& self, requires_arch) noexcept + { + if constexpr (sizeof(T) <= 4) + { + const auto word = static_cast(__lasx_xvpickve2gr_wu(self.data, 0)); + using unsigned_type = sized_uint_t; + return bit_cast(static_cast(word)); + } + else if constexpr (std::is_signed_v) + return static_cast(__lasx_xvpickve2gr_d(self.data, 0)); + else + return static_cast(__lasx_xvpickve2gr_du(self.data, 0)); + } + + template + XSIMD_INLINE float first(batch const& self, requires_arch) noexcept + { + const auto bits = static_cast(__lasx_xvpickve2gr_w(detail::lasx_to_int(self), 0)); + return bit_cast(bits); + } + + template + XSIMD_INLINE double first(batch const& self, requires_arch) noexcept + { + const auto bits = static_cast(__lasx_xvpickve2gr_du(detail::lasx_to_int(self), 0)); + return bit_cast(bits); + } + + template + XSIMD_INLINE std::complex first(batch, A> const& self, requires_arch) noexcept + { + return { first(self.real(), lasx {}), first(self.imag(), lasx {}) }; + } + + // horizontal add of rows + template + XSIMD_INLINE batch haddp(batch const* row, requires_arch) noexcept + { + __m256 abcd = detail::lasx_hadd_pair( + detail::lasx_hadd_pair(row[0].data, row[1].data), + detail::lasx_hadd_pair(row[2].data, row[3].data)); + __m256 efgh = detail::lasx_hadd_pair( + detail::lasx_hadd_pair(row[4].data, row[5].data), + detail::lasx_hadd_pair(row[6].data, row[7].data)); + const __m256i lo = __lasx_xvpermi_q(bit_cast<__m256i>(efgh), bit_cast<__m256i>(abcd), 0x30); + const __m256i hi = __lasx_xvpermi_q(bit_cast<__m256i>(efgh), bit_cast<__m256i>(abcd), 0x21); + return __lasx_xvfadd_s(bit_cast<__m256>(lo), bit_cast<__m256>(hi)); + } + + template + XSIMD_INLINE batch haddp(batch const* row, requires_arch) noexcept + { + const __m256d ab = detail::lasx_hadd_pair(row[0].data, row[1].data); + const __m256d cd = detail::lasx_hadd_pair(row[2].data, row[3].data); + const __m256i lo = __lasx_xvpermi_q(bit_cast<__m256i>(cd), bit_cast<__m256i>(ab), 0x30); + const __m256i hi = __lasx_xvpermi_q(bit_cast<__m256i>(cd), bit_cast<__m256i>(ab), 0x21); + return __lasx_xvfadd_d(bit_cast<__m256d>(lo), bit_cast<__m256d>(hi)); + } + + // load + template >> + XSIMD_INLINE batch load_unaligned(T const* mem, convert, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvld(mem, 0)); + } + + template >> + XSIMD_INLINE batch load_aligned(T const* mem, convert, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvld(mem, 0)); + } + + // load/store complex helpers + namespace detail + { + template + XSIMD_INLINE batch, A> load_complex(batch const& first_chunk, batch const& second_chunk, requires_arch) noexcept + { + __m256i real; + __m256i imag; + if constexpr (sizeof(T) == 4) + { + real = __lasx_xvpickev_w(lasx_to_int(second_chunk), lasx_to_int(first_chunk)); + imag = __lasx_xvpickod_w(lasx_to_int(second_chunk), lasx_to_int(first_chunk)); + } + else + { + real = __lasx_xvpickev_d(lasx_to_int(second_chunk), lasx_to_int(first_chunk)); + imag = __lasx_xvpickod_d(lasx_to_int(second_chunk), lasx_to_int(first_chunk)); + } + // xvpick* operates independently on each 128-bit lane; restore + // the linear element order expected by batch>. + real = __lasx_xvpermi_d(real, 0xd8); + imag = __lasx_xvpermi_d(imag, 0xd8); + return { lasx_from_int(real), lasx_from_int(imag) }; + } + + template + XSIMD_INLINE batch complex_low(batch, A> const& self, requires_arch) noexcept + { + __m256i lo; + __m256i hi; + if constexpr (sizeof(T) == 4) + { + lo = __lasx_xvilvl_w(lasx_to_int(self.imag()), lasx_to_int(self.real())); + hi = __lasx_xvilvh_w(lasx_to_int(self.imag()), lasx_to_int(self.real())); + } + else + { + lo = __lasx_xvilvl_d(lasx_to_int(self.imag()), lasx_to_int(self.real())); + hi = __lasx_xvilvh_d(lasx_to_int(self.imag()), lasx_to_int(self.real())); + } + return lasx_from_int(__lasx_xvpermi_q(hi, lo, 0x20)); + } + + template + XSIMD_INLINE batch complex_high(batch, A> const& self, requires_arch) noexcept + { + __m256i lo; + __m256i hi; + if constexpr (sizeof(T) == 4) + { + lo = __lasx_xvilvl_w(lasx_to_int(self.imag()), lasx_to_int(self.real())); + hi = __lasx_xvilvh_w(lasx_to_int(self.imag()), lasx_to_int(self.real())); + } + else + { + lo = __lasx_xvilvl_d(lasx_to_int(self.imag()), lasx_to_int(self.real())); + hi = __lasx_xvilvh_d(lasx_to_int(self.imag()), lasx_to_int(self.real())); + } + return lasx_from_int(__lasx_xvpermi_q(hi, lo, 0x31)); + } + } + + // max/min + template >> + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lasx_xvmax_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvmax_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvmax_w(self.data, other.data); + else + return __lasx_xvmax_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lasx_xvmax_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvmax_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvmax_wu(self.data, other.data); + else + return __lasx_xvmax_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept + { + const __m256i cond = __lasx_xvfcmp_clt_s(self.data, other.data); + return detail::lasx_from_int(__lasx_xvbitsel_v(detail::lasx_to_int(self), detail::lasx_to_int(other), cond)); + } + + template + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept + { + const __m256i cond = __lasx_xvfcmp_clt_d(self.data, other.data); + return detail::lasx_from_int(__lasx_xvbitsel_v(detail::lasx_to_int(self), detail::lasx_to_int(other), cond)); + } + + template >> + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lasx_xvmin_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvmin_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvmin_w(self.data, other.data); + else + return __lasx_xvmin_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lasx_xvmin_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvmin_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvmin_wu(self.data, other.data); + else + return __lasx_xvmin_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + const __m256i cond = __lasx_xvfcmp_clt_s(other.data, self.data); + return detail::lasx_from_int(__lasx_xvbitsel_v(detail::lasx_to_int(self), detail::lasx_to_int(other), cond)); + } + + template + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + const __m256i cond = __lasx_xvfcmp_clt_d(other.data, self.data); + return detail::lasx_from_int(__lasx_xvbitsel_v(detail::lasx_to_int(self), detail::lasx_to_int(other), cond)); + } + + // mul/neg + template >> + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvmul_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvmul_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvmul_w(self.data, other.data); + else + return __lasx_xvmul_d(self.data, other.data); + } + + template + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfmul_s(self.data, other.data); + } + + template + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfmul_d(self.data, other.data); + } + + template >> + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvneg_b(self.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvneg_h(self.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvneg_w(self.data); + else + return __lasx_xvneg_d(self.data); + } + + template + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + const auto sign_bit = bit_cast(std::uint32_t(1) << 31); + const __m256i sign = __lasx_xvreplgr2vr_w(sign_bit); + return detail::lasx_from_int(__lasx_xvxor_v(detail::lasx_to_int(self), sign)); + } + + template + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + const auto sign_bit = bit_cast(std::uint64_t(1) << 63); + const __m256i sign = __lasx_xvreplgr2vr_d(sign_bit); + return detail::lasx_from_int(__lasx_xvxor_v(detail::lasx_to_int(self), sign)); + } + + // rsqrt/sqrt + template + XSIMD_INLINE batch rsqrt(batch const& self, requires_arch) noexcept + { + return __lasx_xvfrsqrt_s(self.data); + } + + template + XSIMD_INLINE batch rsqrt(batch const& self, requires_arch) noexcept + { + return __lasx_xvfrsqrt_d(self.data); + } + + template + XSIMD_INLINE batch sqrt(batch const& self, requires_arch) noexcept + { + return __lasx_xvfsqrt_s(self.data); + } + + template + XSIMD_INLINE batch sqrt(batch const& self, requires_arch) noexcept + { + return __lasx_xvfsqrt_d(self.data); + } + + // isnan + template + XSIMD_INLINE batch_bool isnan(batch const& self, requires_arch) noexcept + { + return __lasx_xvfcmp_cun_s(self.data, self.data); + } + + template + XSIMD_INLINE batch_bool isnan(batch const& self, requires_arch) noexcept + { + return __lasx_xvfcmp_cun_d(self.data, self.data); + } + + // select + template + XSIMD_INLINE batch select(batch_bool const& cond, batch const& true_br, batch const& false_br, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvbitsel_v(detail::lasx_to_int(false_br), detail::lasx_to_int(true_br), cond.data)); + } + + template + XSIMD_INLINE batch select(batch_bool_constant const&, batch const& true_br, batch const& false_br, requires_arch) noexcept + { + return select(batch_bool { Values... }, true_br, false_br, lasx {}); + } + + // set + template + XSIMD_INLINE batch set(batch const&, requires_arch, Values... values) noexcept + { + static_assert(sizeof...(Values) == batch::size, "consistent init"); + using vector_type = typename detail::lasx_set_vector::type; + const vector_type vector = { static_cast(values)... }; + return bit_cast::register_type>(vector); + } + + template + XSIMD_INLINE batch, A> set(batch, A> const&, requires_arch, Values... values) noexcept + { + return { set(batch {}, lasx {}, values.real()...), + set(batch {}, lasx {}, values.imag()...) }; + } + + template + XSIMD_INLINE batch_bool set(batch_bool const&, requires_arch, Values... values) noexcept + { + using value_type = sized_uint_t; + return set(batch {}, lasx {}, static_cast(values ? ~value_type(0) : value_type(0))...).data; + } + + // byte slides + // xvbsll/xvbsrl operate independently on each 128-bit lane, so the + // adjacent lane is permuted in to supply or receive the carried bytes. + template + XSIMD_INLINE batch slide_left(batch const& self, requires_arch) noexcept + { + static_assert(N <= 32, "invalid byte slide"); + const __m256i bits = detail::lasx_to_int(self); + const __m256i zero = __lasx_xvldi(0); + if constexpr (N == 0) + return self; + else if constexpr (N < 16) + { + const __m256i shifted = __lasx_xvbsll_v(bits, N); + const __m256i previous = __lasx_xvpermi_q(bits, zero, 0x20); + const __m256i carry = __lasx_xvbsrl_v(previous, 16 - N); + return detail::lasx_from_int(__lasx_xvor_v(shifted, carry)); + } + else if constexpr (N == 16) + return detail::lasx_from_int(__lasx_xvpermi_q(bits, zero, 0x20)); + else if constexpr (N < 32) + { + const __m256i previous = __lasx_xvpermi_q(bits, zero, 0x20); + return detail::lasx_from_int(__lasx_xvbsll_v(previous, N - 16)); + } + else + return detail::lasx_from_int(zero); + } + + template + XSIMD_INLINE batch slide_right(batch const& self, requires_arch) noexcept + { + static_assert(N <= 32, "invalid byte slide"); + const __m256i bits = detail::lasx_to_int(self); + const __m256i zero = __lasx_xvldi(0); + if constexpr (N == 0) + return self; + else if constexpr (N < 16) + { + const __m256i shifted = __lasx_xvbsrl_v(bits, N); + const __m256i next = __lasx_xvpermi_q(zero, bits, 0x31); + const __m256i carry = __lasx_xvbsll_v(next, 16 - N); + return detail::lasx_from_int(__lasx_xvor_v(shifted, carry)); + } + else if constexpr (N == 16) + return detail::lasx_from_int(__lasx_xvpermi_q(zero, bits, 0x31)); + else if constexpr (N < 32) + { + const __m256i next = __lasx_xvpermi_q(zero, bits, 0x31); + return detail::lasx_from_int(__lasx_xvbsrl_v(next, N - 16)); + } + else + return detail::lasx_from_int(zero); + } + + // store + template >> + XSIMD_INLINE void store_unaligned(T* mem, batch const& self, requires_arch) noexcept + { + __lasx_xvst(detail::lasx_to_int(self), mem, 0); + } + + template >> + XSIMD_INLINE void store_aligned(T* mem, batch const& self, requires_arch) noexcept + { + __lasx_xvst(detail::lasx_to_int(self), mem, 0); + } + + // sub + template >> + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvsub_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvsub_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvsub_w(self.data, other.data); + else + return __lasx_xvsub_d(self.data, other.data); + } + + template + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfsub_s(self.data, other.data); + } + + template + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfsub_d(self.data, other.data); + } + + // zip + // xvilvl/xvilvh interleave within 128-bit lanes; xvpermi_q assembles + // the full-width lower or upper half of the logical 256-bit vectors. + template + XSIMD_INLINE batch zip_lo(batch const& self, batch const& other, requires_arch) noexcept + { + __m256i lo; + __m256i hi; + if constexpr (sizeof(T) == 1) + { + lo = __lasx_xvilvl_b(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_b(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + else if constexpr (sizeof(T) == 2) + { + lo = __lasx_xvilvl_h(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_h(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + else if constexpr (sizeof(T) == 4) + { + lo = __lasx_xvilvl_w(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_w(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + else + { + lo = __lasx_xvilvl_d(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_d(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + return detail::lasx_from_int(__lasx_xvpermi_q(hi, lo, 0x20)); + } + + template + XSIMD_INLINE batch zip_hi(batch const& self, batch const& other, requires_arch) noexcept + { + __m256i lo; + __m256i hi; + if constexpr (sizeof(T) == 1) + { + lo = __lasx_xvilvl_b(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_b(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + else if constexpr (sizeof(T) == 2) + { + lo = __lasx_xvilvl_h(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_h(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + else if constexpr (sizeof(T) == 4) + { + lo = __lasx_xvilvl_w(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_w(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + else + { + lo = __lasx_xvilvl_d(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_d(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + return detail::lasx_from_int(__lasx_xvpermi_q(hi, lo, 0x31)); + } + } +} + +#endif diff --git a/include/xsimd/arch/xsimd_lsx.hpp b/include/xsimd/arch/xsimd_lsx.hpp index 60f0ba7d5..af86b2b01 100644 --- a/include/xsimd/arch/xsimd_lsx.hpp +++ b/include/xsimd/arch/xsimd_lsx.hpp @@ -14,9 +14,8 @@ #include "../types/xsimd_batch_constant.hpp" #include "../types/xsimd_lsx_register.hpp" +#include "../types/xsimd_utils.hpp" -#include -#include #include #include #include @@ -28,544 +27,887 @@ namespace xsimd { namespace detail { - template - XSIMD_INLINE To loongarch_bit_cast(From const& value) noexcept + template + struct lsx_set_vector { - static_assert(sizeof(To) == sizeof(From), "incompatible vector sizes"); - To result; - __builtin_memcpy(&result, &value, sizeof(result)); - return result; - } - - template - using loongarch_unsigned_register_t = typename batch, A>::register_type; + typedef T type __attribute__((vector_size(16))); + }; template - XSIMD_INLINE loongarch_unsigned_register_t loongarch_to_bits(batch const& value) noexcept + XSIMD_INLINE __m128i lsx_to_int(batch const& value) noexcept { - return loongarch_bit_cast>(value.data); + return bit_cast<__m128i>(value.data); } template - XSIMD_INLINE typename batch::register_type loongarch_from_bits(loongarch_unsigned_register_t const& value) noexcept + XSIMD_INLINE typename batch::register_type lsx_from_int(__m128i value) noexcept { - return loongarch_bit_cast::register_type>(value); + return bit_cast::register_type>(value); } - template - XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + template + XSIMD_INLINE std::uint32_t lsx_mask(__m128i value) noexcept { - typename batch::register_type result {}; - for (std::size_t i = 0; i < batch::size; ++i) + if constexpr (sizeof(T) == 1) + { + return static_cast(__lsx_vpickve2gr_w(__lsx_vmskltz_b(value), 0)); + } + else if constexpr (sizeof(T) == 2) + { + return static_cast(__lsx_vpickve2gr_w(__lsx_vmskltz_h(value), 0)); + } + else if constexpr (sizeof(T) == 4) + { + return static_cast(__lsx_vpickve2gr_w(__lsx_vmskltz_w(value), 0)); + } + else { - result[i] = static_cast(self.data[i]); + return static_cast(__lsx_vpickve2gr_w(__lsx_vmskltz_d(value), 0)); } - return result; } + // fast_cast template - XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept { - typename batch::register_type result {}; - for (std::size_t i = 0; i < batch::size; ++i) - { - result[i] = static_cast(self.data[i]); - } - return result; + return __lsx_vffint_s_w(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vffint_s_wu(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vffint_d_l(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vffint_d_lu(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vftintrz_w_s(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vftintrz_wu_s(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vftintrz_l_d(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vftintrz_lu_d(self.data); } } // abs - template >> - XSIMD_INLINE batch abs(batch const& self, requires_arch) noexcept + template + XSIMD_INLINE batch abs(batch const& self, requires_arch) noexcept { - typename batch::register_type result {}; - for (std::size_t i = 0; i < batch::size; ++i) - { - result[i] = std::abs(self.data[i]); - } - return result; + return detail::lsx_from_int(__lsx_vbitclri_w(detail::lsx_to_int(self), 31)); + } + + template + XSIMD_INLINE batch abs(batch const& self, requires_arch) noexcept + { + return detail::lsx_from_int(__lsx_vbitclri_d(detail::lsx_to_int(self), 63)); } // add - template >> - XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + template >> + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept { - return self.data + other.data; + if constexpr (sizeof(T) == 1) + return __lsx_vadd_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vadd_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vadd_w(self.data, other.data); + else + return __lsx_vadd_d(self.data, other.data); } - // all + template + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfadd_s(self.data, other.data); + } + + template + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfadd_d(self.data, other.data); + } + + // all/any template - XSIMD_INLINE bool all(batch_bool const& self, requires_arch) noexcept + XSIMD_INLINE bool all(batch_bool const& self, requires_arch) noexcept { - for (std::size_t i = 0; i < batch_bool::size; ++i) - { - if (self.data[i] == 0) - { - return false; - } - } - return true; + constexpr std::uint32_t all_bits = (std::uint32_t(1) << batch_bool::size) - 1; + return detail::lsx_mask(self.data) == all_bits; } - // any template - XSIMD_INLINE bool any(batch_bool const& self, requires_arch) noexcept + XSIMD_INLINE bool any(batch_bool const& self, requires_arch) noexcept { - for (std::size_t i = 0; i < batch_bool::size; ++i) - { - if (self.data[i] != 0) - { - return true; - } - } - return false; + return detail::lsx_mask(self.data) != 0; } // batch_bool_cast template - XSIMD_INLINE batch_bool batch_bool_cast(batch_bool const& self, batch_bool const&, requires_arch) noexcept + XSIMD_INLINE batch_bool batch_bool_cast(batch_bool const& self, batch_bool const&, requires_arch) noexcept { - using result_type = typename batch_bool::register_type; - return detail::loongarch_bit_cast(self.data); + return self.data; } // bitwise operations template - XSIMD_INLINE batch bitwise_and(batch const& self, batch const& other, requires_arch) noexcept + XSIMD_INLINE batch bitwise_and(batch const& self, batch const& other, requires_arch) noexcept { - auto bits = detail::loongarch_to_bits(self) & detail::loongarch_to_bits(other); - return detail::loongarch_from_bits(bits); + return detail::lsx_from_int(__lsx_vand_v(detail::lsx_to_int(self), detail::lsx_to_int(other))); } template - XSIMD_INLINE batch_bool bitwise_and(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + XSIMD_INLINE batch_bool bitwise_and(batch_bool const& self, batch_bool const& other, requires_arch) noexcept { - return self.data & other.data; + return __lsx_vand_v(self.data, other.data); } template - XSIMD_INLINE batch bitwise_andnot(batch const& self, batch const& other, requires_arch) noexcept + XSIMD_INLINE batch bitwise_andnot(batch const& self, batch const& other, requires_arch) noexcept { - auto bits = detail::loongarch_to_bits(self) & ~detail::loongarch_to_bits(other); - return detail::loongarch_from_bits(bits); + return detail::lsx_from_int(__lsx_vandn_v(detail::lsx_to_int(other), detail::lsx_to_int(self))); } template - XSIMD_INLINE batch_bool bitwise_andnot(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + XSIMD_INLINE batch_bool bitwise_andnot(batch_bool const& self, batch_bool const& other, requires_arch) noexcept { - return self.data & ~other.data; + return __lsx_vandn_v(other.data, self.data); } template - XSIMD_INLINE batch bitwise_not(batch const& self, requires_arch) noexcept + XSIMD_INLINE batch bitwise_not(batch const& self, requires_arch) noexcept { - auto bits = ~detail::loongarch_to_bits(self); - return detail::loongarch_from_bits(bits); + const __m128i bits = detail::lsx_to_int(self); + return detail::lsx_from_int(__lsx_vnor_v(bits, bits)); } template - XSIMD_INLINE batch_bool bitwise_not(batch_bool const& self, requires_arch) noexcept + XSIMD_INLINE batch_bool bitwise_not(batch_bool const& self, requires_arch) noexcept { - return ~self.data; + return __lsx_vnor_v(self.data, self.data); } template - XSIMD_INLINE batch bitwise_or(batch const& self, batch const& other, requires_arch) noexcept + XSIMD_INLINE batch bitwise_or(batch const& self, batch const& other, requires_arch) noexcept { - auto bits = detail::loongarch_to_bits(self) | detail::loongarch_to_bits(other); - return detail::loongarch_from_bits(bits); + return detail::lsx_from_int(__lsx_vor_v(detail::lsx_to_int(self), detail::lsx_to_int(other))); } template - XSIMD_INLINE batch_bool bitwise_or(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + XSIMD_INLINE batch_bool bitwise_or(batch_bool const& self, batch_bool const& other, requires_arch) noexcept { - return self.data | other.data; + return __lsx_vor_v(self.data, other.data); } template - XSIMD_INLINE batch bitwise_xor(batch const& self, batch const& other, requires_arch) noexcept + XSIMD_INLINE batch bitwise_xor(batch const& self, batch const& other, requires_arch) noexcept { - auto bits = detail::loongarch_to_bits(self) ^ detail::loongarch_to_bits(other); - return detail::loongarch_from_bits(bits); + return detail::lsx_from_int(__lsx_vxor_v(detail::lsx_to_int(self), detail::lsx_to_int(other))); } template - XSIMD_INLINE batch_bool bitwise_xor(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + XSIMD_INLINE batch_bool bitwise_xor(batch_bool const& self, batch_bool const& other, requires_arch) noexcept { - return self.data ^ other.data; + return __lsx_vxor_v(self.data, other.data); } template - XSIMD_INLINE batch bitwise_cast(batch const& self, batch const&, requires_arch) noexcept + XSIMD_INLINE batch bitwise_cast(batch const& self, batch const&, requires_arch) noexcept { - using result_type = typename batch::register_type; - return detail::loongarch_bit_cast(self.data); + return bit_cast::register_type>(self.data); } + // shifts template >> - XSIMD_INLINE batch bitwise_lshift(batch const& self, std::int32_t other, requires_arch) noexcept + XSIMD_INLINE batch bitwise_lshift(batch const& self, std::int32_t other, requires_arch) noexcept { - return self.data << other; + if constexpr (sizeof(T) == 1) + return __lsx_vsll_b(self.data, __lsx_vreplgr2vr_b(other)); + else if constexpr (sizeof(T) == 2) + return __lsx_vsll_h(self.data, __lsx_vreplgr2vr_h(other)); + else if constexpr (sizeof(T) == 4) + return __lsx_vsll_w(self.data, __lsx_vreplgr2vr_w(other)); + else + return __lsx_vsll_d(self.data, __lsx_vreplgr2vr_d(other)); } template >> - XSIMD_INLINE batch bitwise_rshift(batch const& self, std::int32_t other, requires_arch) noexcept + XSIMD_INLINE batch bitwise_rshift(batch const& self, std::int32_t other, requires_arch) noexcept { - return self.data >> other; + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lsx_vsra_b(self.data, __lsx_vreplgr2vr_b(other)); + else if constexpr (sizeof(T) == 2) + return __lsx_vsra_h(self.data, __lsx_vreplgr2vr_h(other)); + else if constexpr (sizeof(T) == 4) + return __lsx_vsra_w(self.data, __lsx_vreplgr2vr_w(other)); + else + return __lsx_vsra_d(self.data, __lsx_vreplgr2vr_d(other)); + } + else + { + if constexpr (sizeof(T) == 1) + return __lsx_vsrl_b(self.data, __lsx_vreplgr2vr_b(other)); + else if constexpr (sizeof(T) == 2) + return __lsx_vsrl_h(self.data, __lsx_vreplgr2vr_h(other)); + else if constexpr (sizeof(T) == 4) + return __lsx_vsrl_w(self.data, __lsx_vreplgr2vr_w(other)); + else + return __lsx_vsrl_d(self.data, __lsx_vreplgr2vr_d(other)); + } } // div - template >> - XSIMD_INLINE batch div(batch const& self, batch const& other, requires_arch) noexcept + template + XSIMD_INLINE batch div(batch const& self, batch const& other, requires_arch) noexcept { - return self.data / other.data; + return __lsx_vfdiv_s(self.data, other.data); + } + + template + XSIMD_INLINE batch div(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfdiv_d(self.data, other.data); } // broadcast - template >> - XSIMD_INLINE batch broadcast(T value, requires_arch) noexcept + template >> + XSIMD_INLINE batch broadcast(T value, requires_arch) noexcept { - typename batch::register_type result {}; - for (std::size_t i = 0; i < batch::size; ++i) - { - result[i] = value; - } - return result; + if constexpr (sizeof(T) == 1) + return __lsx_vreplgr2vr_b(static_cast(value)); + else if constexpr (sizeof(T) == 2) + return __lsx_vreplgr2vr_h(static_cast(value)); + else if constexpr (sizeof(T) == 4) + return __lsx_vreplgr2vr_w(bit_cast>(value)); + else + return __lsx_vreplgr2vr_d(bit_cast>(value)); + } + + template + XSIMD_INLINE batch broadcast(float value, requires_arch) noexcept + { + const auto bits = bit_cast(value); + return bit_cast<__m128>(__lsx_vreplgr2vr_w(bits)); + } + + template + XSIMD_INLINE batch broadcast(double value, requires_arch) noexcept + { + const auto bits = bit_cast(value); + return bit_cast<__m128d>(__lsx_vreplgr2vr_d(static_cast(bits))); } // comparisons - template - XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept + template >> + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept { - using result_type = typename batch_bool::register_type; - return detail::loongarch_bit_cast(self.data == other.data); + if constexpr (sizeof(T) == 1) + return __lsx_vseq_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vseq_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vseq_w(self.data, other.data); + else + return __lsx_vseq_d(self.data, other.data); } - template - XSIMD_INLINE batch_bool eq(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + template + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept { - return self.data == other.data; + return __lsx_vfcmp_ceq_s(self.data, other.data); } - template - XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + template + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept { - using result_type = typename batch_bool::register_type; - return detail::loongarch_bit_cast(self.data < other.data); + return __lsx_vfcmp_ceq_d(self.data, other.data); } template - XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + XSIMD_INLINE batch_bool eq(batch_bool const& self, batch_bool const& other, requires_arch) noexcept { - using result_type = typename batch_bool::register_type; - return detail::loongarch_bit_cast(self.data <= other.data); + if constexpr (sizeof(T) == 1) + return __lsx_vseq_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vseq_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vseq_w(self.data, other.data); + else + return __lsx_vseq_d(self.data, other.data); } - template - XSIMD_INLINE batch_bool gt(batch const& self, batch const& other, requires_arch) noexcept + template >> + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lsx_vslt_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vslt_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vslt_w(self.data, other.data); + else + return __lsx_vslt_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lsx_vslt_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vslt_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vslt_wu(self.data, other.data); + else + return __lsx_vslt_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfcmp_clt_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept { - using result_type = typename batch_bool::register_type; - return detail::loongarch_bit_cast(self.data > other.data); + return __lsx_vfcmp_clt_d(self.data, other.data); + } + + template >> + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lsx_vsle_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vsle_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vsle_w(self.data, other.data); + else + return __lsx_vsle_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lsx_vsle_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vsle_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vsle_wu(self.data, other.data); + else + return __lsx_vsle_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfcmp_cle_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfcmp_cle_d(self.data, other.data); } template - XSIMD_INLINE batch_bool ge(batch const& self, batch const& other, requires_arch) noexcept + XSIMD_INLINE batch_bool gt(batch const& self, batch const& other, requires_arch) noexcept { - using result_type = typename batch_bool::register_type; - return detail::loongarch_bit_cast(self.data >= other.data); + return lt(other, self, lsx {}); } template - XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + XSIMD_INLINE batch_bool ge(batch const& self, batch const& other, requires_arch) noexcept + { + return le(other, self, lsx {}); + } + + template >> + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + { + const __m128i equal = eq(self, other, lsx {}).data; + return __lsx_vnor_v(equal, equal); + } + + template + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfcmp_cune_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept { - return bitwise_not(eq(self, other, loongarch {}), loongarch {}); + return __lsx_vfcmp_cune_d(self.data, other.data); } template - XSIMD_INLINE batch_bool neq(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + XSIMD_INLINE batch_bool neq(batch_bool const& self, batch_bool const& other, requires_arch) noexcept { - return self.data != other.data; + return __lsx_vxor_v(self.data, other.data); } // first - template >> - XSIMD_INLINE T first(batch const& self, requires_arch) noexcept + template >> + XSIMD_INLINE T first(batch const& self, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_signed_v) + return static_cast(__lsx_vpickve2gr_b(self.data, 0)); + else + return static_cast(__lsx_vpickve2gr_bu(self.data, 0)); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_signed_v) + return static_cast(__lsx_vpickve2gr_h(self.data, 0)); + else + return static_cast(__lsx_vpickve2gr_hu(self.data, 0)); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_signed_v) + return static_cast(__lsx_vpickve2gr_w(self.data, 0)); + else + return static_cast(__lsx_vpickve2gr_wu(self.data, 0)); + } + else + { + if constexpr (std::is_signed_v) + return static_cast(__lsx_vpickve2gr_d(self.data, 0)); + else + return static_cast(__lsx_vpickve2gr_du(self.data, 0)); + } + } + + template + XSIMD_INLINE float first(batch const& self, requires_arch) noexcept { - return self.data[0]; + const auto bits = static_cast(__lsx_vpickve2gr_w(detail::lsx_to_int(self), 0)); + return bit_cast(bits); + } + + template + XSIMD_INLINE double first(batch const& self, requires_arch) noexcept + { + const auto bits = static_cast(__lsx_vpickve2gr_du(detail::lsx_to_int(self), 0)); + return bit_cast(bits); } template - XSIMD_INLINE std::complex first(batch, A> const& self, requires_arch) noexcept + XSIMD_INLINE std::complex first(batch, A> const& self, requires_arch) noexcept { - return { self.real().data[0], self.imag().data[0] }; + return { first(self.real(), lsx {}), first(self.imag(), lsx {}) }; } // horizontal add of rows - template - XSIMD_INLINE batch haddp(batch const* row, requires_arch) noexcept + template + XSIMD_INLINE batch haddp(batch const* row, requires_arch) noexcept { - typename batch::register_type result {}; - for (std::size_t i = 0; i < batch::size; ++i) - { - T value = T(0); - for (std::size_t j = 0; j < batch::size; ++j) - { - value += row[i].data[j]; - } - result[i] = value; - } - return result; + const __m128i ab_lo = __lsx_vilvl_w(detail::lsx_to_int(row[1]), detail::lsx_to_int(row[0])); + const __m128i ab_hi = __lsx_vilvh_w(detail::lsx_to_int(row[1]), detail::lsx_to_int(row[0])); + const __m128 ab = __lsx_vfadd_s(bit_cast<__m128>(ab_lo), bit_cast<__m128>(ab_hi)); + const __m128i cd_lo = __lsx_vilvl_w(detail::lsx_to_int(row[3]), detail::lsx_to_int(row[2])); + const __m128i cd_hi = __lsx_vilvh_w(detail::lsx_to_int(row[3]), detail::lsx_to_int(row[2])); + const __m128 cd = __lsx_vfadd_s(bit_cast<__m128>(cd_lo), bit_cast<__m128>(cd_hi)); + const __m128i lo = __lsx_vilvl_d(bit_cast<__m128i>(cd), bit_cast<__m128i>(ab)); + const __m128i hi = __lsx_vilvh_d(bit_cast<__m128i>(cd), bit_cast<__m128i>(ab)); + return __lsx_vfadd_s(bit_cast<__m128>(lo), bit_cast<__m128>(hi)); + } + + template + XSIMD_INLINE batch haddp(batch const* row, requires_arch) noexcept + { + const __m128i lo = __lsx_vilvl_d(detail::lsx_to_int(row[1]), detail::lsx_to_int(row[0])); + const __m128i hi = __lsx_vilvh_d(detail::lsx_to_int(row[1]), detail::lsx_to_int(row[0])); + return __lsx_vfadd_d(bit_cast<__m128d>(lo), bit_cast<__m128d>(hi)); } // load template >> - XSIMD_INLINE batch load_unaligned(T const* mem, convert, requires_arch) noexcept + XSIMD_INLINE batch load_unaligned(T const* mem, convert, requires_arch) noexcept { - typename batch::register_type result; - __builtin_memcpy(&result, mem, sizeof(result)); - return result; + return detail::lsx_from_int(__lsx_vld(mem, 0)); } template >> - XSIMD_INLINE batch load_aligned(T const* mem, convert, requires_arch) noexcept + XSIMD_INLINE batch load_aligned(T const* mem, convert, requires_arch) noexcept { - return load_unaligned(mem, convert {}, loongarch {}); + return detail::lsx_from_int(__lsx_vld(mem, 0)); } // load/store complex helpers namespace detail { template - XSIMD_INLINE batch, A> load_complex(batch const& first_chunk, batch const& second_chunk, requires_arch) noexcept + XSIMD_INLINE batch, A> load_complex(batch const& first_chunk, batch const& second_chunk, requires_arch) noexcept { - constexpr std::size_t size = batch::size; - std::array real {}; - std::array imag {}; - for (std::size_t i = 0; i < size; ++i) + __m128i real; + __m128i imag; + if constexpr (sizeof(T) == 4) { - const std::size_t real_index = 2 * i; - const std::size_t imag_index = real_index + 1; - real[i] = real_index < size ? first_chunk.data[real_index] : second_chunk.data[real_index - size]; - imag[i] = imag_index < size ? first_chunk.data[imag_index] : second_chunk.data[imag_index - size]; + real = __lsx_vpickev_w(lsx_to_int(second_chunk), lsx_to_int(first_chunk)); + imag = __lsx_vpickod_w(lsx_to_int(second_chunk), lsx_to_int(first_chunk)); } - return { load_unaligned(real.data(), convert {}, loongarch {}), - load_unaligned(imag.data(), convert {}, loongarch {}) }; + else + { + real = __lsx_vpickev_d(lsx_to_int(second_chunk), lsx_to_int(first_chunk)); + imag = __lsx_vpickod_d(lsx_to_int(second_chunk), lsx_to_int(first_chunk)); + } + return { lsx_from_int(real), lsx_from_int(imag) }; } template - XSIMD_INLINE batch complex_low(batch, A> const& self, requires_arch) noexcept + XSIMD_INLINE batch complex_low(batch, A> const& self, requires_arch) noexcept { - constexpr std::size_t size = batch::size; - std::array result {}; - for (std::size_t i = 0; i < size; ++i) - { - const std::size_t source = i / 2; - result[i] = (i % 2 == 0) ? self.real().data[source] : self.imag().data[source]; - } - return load_unaligned(result.data(), convert {}, loongarch {}); + if constexpr (sizeof(T) == 4) + return lsx_from_int(__lsx_vilvl_w(lsx_to_int(self.imag()), lsx_to_int(self.real()))); + else + return lsx_from_int(__lsx_vilvl_d(lsx_to_int(self.imag()), lsx_to_int(self.real()))); } template - XSIMD_INLINE batch complex_high(batch, A> const& self, requires_arch) noexcept + XSIMD_INLINE batch complex_high(batch, A> const& self, requires_arch) noexcept { - constexpr std::size_t size = batch::size; - std::array result {}; - for (std::size_t i = 0; i < size; ++i) - { - const std::size_t interleaved_index = size + i; - const std::size_t source = interleaved_index / 2; - result[i] = (interleaved_index % 2 == 0) ? self.real().data[source] : self.imag().data[source]; - } - return load_unaligned(result.data(), convert {}, loongarch {}); + if constexpr (sizeof(T) == 4) + return lsx_from_int(__lsx_vilvh_w(lsx_to_int(self.imag()), lsx_to_int(self.real()))); + else + return lsx_from_int(__lsx_vilvh_d(lsx_to_int(self.imag()), lsx_to_int(self.real()))); } } // max/min - template - XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept + template >> + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept { - typename batch::register_type result {}; - for (std::size_t i = 0; i < batch::size; ++i) + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lsx_vmax_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vmax_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vmax_w(self.data, other.data); + else + return __lsx_vmax_d(self.data, other.data); + } + else { - result[i] = self.data[i] < other.data[i] ? other.data[i] : self.data[i]; + if constexpr (sizeof(T) == 1) + return __lsx_vmax_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vmax_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vmax_wu(self.data, other.data); + else + return __lsx_vmax_du(self.data, other.data); } - return result; } - template - XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + template + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept + { + const __m128i cond = __lsx_vfcmp_clt_s(self.data, other.data); + return detail::lsx_from_int(__lsx_vbitsel_v(detail::lsx_to_int(self), detail::lsx_to_int(other), cond)); + } + + template + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept { - typename batch::register_type result {}; - for (std::size_t i = 0; i < batch::size; ++i) + const __m128i cond = __lsx_vfcmp_clt_d(self.data, other.data); + return detail::lsx_from_int(__lsx_vbitsel_v(detail::lsx_to_int(self), detail::lsx_to_int(other), cond)); + } + + template >> + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lsx_vmin_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vmin_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vmin_w(self.data, other.data); + else + return __lsx_vmin_d(self.data, other.data); + } + else { - result[i] = self.data[i] < other.data[i] ? self.data[i] : other.data[i]; + if constexpr (sizeof(T) == 1) + return __lsx_vmin_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vmin_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vmin_wu(self.data, other.data); + else + return __lsx_vmin_du(self.data, other.data); } - return result; + } + + template + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + const __m128i cond = __lsx_vfcmp_clt_s(other.data, self.data); + return detail::lsx_from_int(__lsx_vbitsel_v(detail::lsx_to_int(self), detail::lsx_to_int(other), cond)); + } + + template + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + const __m128i cond = __lsx_vfcmp_clt_d(other.data, self.data); + return detail::lsx_from_int(__lsx_vbitsel_v(detail::lsx_to_int(self), detail::lsx_to_int(other), cond)); } // mul/neg - template - XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept + template >> + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept { - return self.data * other.data; + if constexpr (sizeof(T) == 1) + return __lsx_vmul_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vmul_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vmul_w(self.data, other.data); + else + return __lsx_vmul_d(self.data, other.data); } - template - XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + template + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept { - return -self.data; + return __lsx_vfmul_s(self.data, other.data); } - // rsqrt - template >> - XSIMD_INLINE batch rsqrt(batch const& self, requires_arch) noexcept + template + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept { - typename batch::register_type result {}; - for (std::size_t i = 0; i < batch::size; ++i) - { - result[i] = T(1) / std::sqrt(self.data[i]); - } - return result; + return __lsx_vfmul_d(self.data, other.data); + } + + template >> + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lsx_vneg_b(self.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vneg_h(self.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vneg_w(self.data); + else + return __lsx_vneg_d(self.data); + } + + template + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + const auto sign_bit = bit_cast(std::uint32_t(1) << 31); + const __m128i sign = __lsx_vreplgr2vr_w(sign_bit); + return detail::lsx_from_int(__lsx_vxor_v(detail::lsx_to_int(self), sign)); + } + + template + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + const auto sign_bit = bit_cast(std::uint64_t(1) << 63); + const __m128i sign = __lsx_vreplgr2vr_d(sign_bit); + return detail::lsx_from_int(__lsx_vxor_v(detail::lsx_to_int(self), sign)); + } + + // rsqrt/sqrt + template + XSIMD_INLINE batch rsqrt(batch const& self, requires_arch) noexcept + { + return __lsx_vfrsqrt_s(self.data); + } + + template + XSIMD_INLINE batch rsqrt(batch const& self, requires_arch) noexcept + { + return __lsx_vfrsqrt_d(self.data); + } + + template + XSIMD_INLINE batch sqrt(batch const& self, requires_arch) noexcept + { + return __lsx_vfsqrt_s(self.data); + } + + template + XSIMD_INLINE batch sqrt(batch const& self, requires_arch) noexcept + { + return __lsx_vfsqrt_d(self.data); } // isnan - template >> - XSIMD_INLINE batch_bool isnan(batch const& self, requires_arch) noexcept + template + XSIMD_INLINE batch_bool isnan(batch const& self, requires_arch) noexcept { - return neq(self, self, loongarch {}); + return __lsx_vfcmp_cun_s(self.data, self.data); + } + + template + XSIMD_INLINE batch_bool isnan(batch const& self, requires_arch) noexcept + { + return __lsx_vfcmp_cun_d(self.data, self.data); } // select template - XSIMD_INLINE batch select(batch_bool const& cond, batch const& true_br, batch const& false_br, requires_arch) noexcept + XSIMD_INLINE batch select(batch_bool const& cond, batch const& true_br, batch const& false_br, requires_arch) noexcept { - using bits_type = detail::loongarch_unsigned_register_t; - auto mask = detail::loongarch_bit_cast(cond.data); - auto true_bits = detail::loongarch_to_bits(true_br); - auto false_bits = detail::loongarch_to_bits(false_br); - return detail::loongarch_from_bits((mask & true_bits) | (~mask & false_bits)); + return detail::lsx_from_int(__lsx_vbitsel_v(detail::lsx_to_int(false_br), detail::lsx_to_int(true_br), cond.data)); } template - XSIMD_INLINE batch select(batch_bool_constant const&, batch const& true_br, batch const& false_br, requires_arch) noexcept + XSIMD_INLINE batch select(batch_bool_constant const&, batch const& true_br, batch const& false_br, requires_arch) noexcept { - return select(batch_bool { Values... }, true_br, false_br, loongarch {}); + return select(batch_bool { Values... }, true_br, false_br, lsx {}); } // set template - XSIMD_INLINE batch set(batch const&, requires_arch, Values... values) noexcept + XSIMD_INLINE batch set(batch const&, requires_arch, Values... values) noexcept { static_assert(sizeof...(Values) == batch::size, "consistent init"); - return typename batch::register_type { static_cast(values)... }; + using vector_type = typename detail::lsx_set_vector::type; + const vector_type vector = { static_cast(values)... }; + return bit_cast::register_type>(vector); } template - XSIMD_INLINE batch, A> set(batch, A> const&, requires_arch, Values... values) noexcept + XSIMD_INLINE batch, A> set(batch, A> const&, requires_arch, Values... values) noexcept { - return batch, A>(set(batch {}, loongarch {}, values.real()...), - set(batch {}, loongarch {}, values.imag()...)); + return { set(batch {}, lsx {}, values.real()...), + set(batch {}, lsx {}, values.imag()...) }; } template - XSIMD_INLINE batch_bool set(batch_bool const&, requires_arch, Values... values) noexcept + XSIMD_INLINE batch_bool set(batch_bool const&, requires_arch, Values... values) noexcept { - static_assert(sizeof...(Values) == batch_bool::size, "consistent init"); using value_type = sized_uint_t; - return typename batch_bool::register_type { static_cast(values ? ~value_type(0) : value_type(0))... }; - } - - // sqrt - template >> - XSIMD_INLINE batch sqrt(batch const& self, requires_arch) noexcept - { - typename batch::register_type result {}; - for (std::size_t i = 0; i < batch::size; ++i) - { - result[i] = std::sqrt(self.data[i]); - } - return result; + return set(batch {}, lsx {}, static_cast(values ? ~value_type(0) : value_type(0))...).data; } // byte slides template - XSIMD_INLINE batch slide_left(batch const& self, requires_arch) noexcept + XSIMD_INLINE batch slide_left(batch const& self, requires_arch) noexcept { - static_assert(N <= A::alignment(), "invalid byte slide"); - std::array input {}; - std::array output {}; - __builtin_memcpy(input.data(), &self.data, input.size()); - for (std::size_t i = N; i < output.size(); ++i) - { - output[i] = input[i - N]; - } - typename batch::register_type result; - __builtin_memcpy(&result, output.data(), output.size()); - return result; + static_assert(N <= 16, "invalid byte slide"); + if constexpr (N == 16) + return detail::lsx_from_int(__lsx_vldi(0)); + else + return detail::lsx_from_int(__lsx_vbsll_v(detail::lsx_to_int(self), N)); } template - XSIMD_INLINE batch slide_right(batch const& self, requires_arch) noexcept + XSIMD_INLINE batch slide_right(batch const& self, requires_arch) noexcept { - static_assert(N <= A::alignment(), "invalid byte slide"); - std::array input {}; - std::array output {}; - __builtin_memcpy(input.data(), &self.data, input.size()); - for (std::size_t i = 0; i + N < output.size(); ++i) - { - output[i] = input[i + N]; - } - typename batch::register_type result; - __builtin_memcpy(&result, output.data(), output.size()); - return result; + static_assert(N <= 16, "invalid byte slide"); + if constexpr (N == 16) + return detail::lsx_from_int(__lsx_vldi(0)); + else + return detail::lsx_from_int(__lsx_vbsrl_v(detail::lsx_to_int(self), N)); } // store template >> - XSIMD_INLINE void store_unaligned(T* mem, batch const& self, requires_arch) noexcept + XSIMD_INLINE void store_unaligned(T* mem, batch const& self, requires_arch) noexcept { - __builtin_memcpy(mem, &self.data, sizeof(self.data)); + __lsx_vst(detail::lsx_to_int(self), mem, 0); } template >> - XSIMD_INLINE void store_aligned(T* mem, batch const& self, requires_arch) noexcept + XSIMD_INLINE void store_aligned(T* mem, batch const& self, requires_arch) noexcept { - store_unaligned(mem, self, loongarch {}); + __lsx_vst(detail::lsx_to_int(self), mem, 0); } // sub - template - XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + template >> + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept { - return self.data - other.data; + if constexpr (sizeof(T) == 1) + return __lsx_vsub_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vsub_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vsub_w(self.data, other.data); + else + return __lsx_vsub_d(self.data, other.data); + } + + template + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfsub_s(self.data, other.data); + } + + template + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfsub_d(self.data, other.data); } // zip template - XSIMD_INLINE batch zip_lo(batch const& self, batch const& other, requires_arch) noexcept + XSIMD_INLINE batch zip_lo(batch const& self, batch const& other, requires_arch) noexcept { - typename batch::register_type result {}; - constexpr std::size_t half = batch::size / 2; - for (std::size_t i = 0; i < half; ++i) - { - result[2 * i] = self.data[i]; - result[2 * i + 1] = other.data[i]; - } - return result; + if constexpr (sizeof(T) == 1) + return detail::lsx_from_int(__lsx_vilvl_b(detail::lsx_to_int(other), detail::lsx_to_int(self))); + else if constexpr (sizeof(T) == 2) + return detail::lsx_from_int(__lsx_vilvl_h(detail::lsx_to_int(other), detail::lsx_to_int(self))); + else if constexpr (sizeof(T) == 4) + return detail::lsx_from_int(__lsx_vilvl_w(detail::lsx_to_int(other), detail::lsx_to_int(self))); + else + return detail::lsx_from_int(__lsx_vilvl_d(detail::lsx_to_int(other), detail::lsx_to_int(self))); } template - XSIMD_INLINE batch zip_hi(batch const& self, batch const& other, requires_arch) noexcept - { - typename batch::register_type result {}; - constexpr std::size_t half = batch::size / 2; - for (std::size_t i = 0; i < half; ++i) - { - result[2 * i] = self.data[i + half]; - result[2 * i + 1] = other.data[i + half]; - } - return result; + XSIMD_INLINE batch zip_hi(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return detail::lsx_from_int(__lsx_vilvh_b(detail::lsx_to_int(other), detail::lsx_to_int(self))); + else if constexpr (sizeof(T) == 2) + return detail::lsx_from_int(__lsx_vilvh_h(detail::lsx_to_int(other), detail::lsx_to_int(self))); + else if constexpr (sizeof(T) == 4) + return detail::lsx_from_int(__lsx_vilvh_w(detail::lsx_to_int(other), detail::lsx_to_int(self))); + else + return detail::lsx_from_int(__lsx_vilvh_d(detail::lsx_to_int(other), detail::lsx_to_int(self))); } } } diff --git a/include/xsimd/config/xsimd_cpu_features_loongarch.hpp b/include/xsimd/config/xsimd_cpu_features_loongarch.hpp index 9ef616177..fd34e60fd 100644 --- a/include/xsimd/config/xsimd_cpu_features_loongarch.hpp +++ b/include/xsimd/config/xsimd_cpu_features_loongarch.hpp @@ -15,11 +15,15 @@ #include "./xsimd_config.hpp" #include "./xsimd_getauxval.hpp" +#include +#include + #if XSIMD_TARGET_LOONGARCH64 && XSIMD_HAVE_LINUX_GETAUXVAL -// HWCAP masks are architecture-specific and asm/hwcap.h is not available -// for every target. +// HWCAP_XXX masks to use on getauxval results. +// Header does not exists on all architectures and masks are architecture +// specific. #include -#endif +#endif // XSIMD_TARGET_LOONGARCH64 && XSIMD_HAVE_LINUX_GETAUXVAL namespace xsimd { diff --git a/include/xsimd/types/xsimd_all_registers.hpp b/include/xsimd/types/xsimd_all_registers.hpp index 8bd27aabf..aae59fc35 100644 --- a/include/xsimd/types/xsimd_all_registers.hpp +++ b/include/xsimd/types/xsimd_all_registers.hpp @@ -34,6 +34,7 @@ #include "./xsimd_neon64_register.hpp" #include "./xsimd_neon_register.hpp" #include "./xsimd_rvv_register.hpp" +#include "./xsimd_lasx_register.hpp" #include "./xsimd_lsx_register.hpp" #include "./xsimd_sse2_register.hpp" #include "./xsimd_sse3_register.hpp" diff --git a/include/xsimd/types/xsimd_lasx_register.hpp b/include/xsimd/types/xsimd_lasx_register.hpp new file mode 100644 index 000000000..cb14331bc --- /dev/null +++ b/include/xsimd/types/xsimd_lasx_register.hpp @@ -0,0 +1,67 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_LASX_REGISTER_HPP +#define XSIMD_LASX_REGISTER_HPP + +#include "./xsimd_lsx_register.hpp" + +#if XSIMD_WITH_LASX && !XSIMD_WITH_LSX +#error "architecture inconsistency: lasx requires lsx" +#endif + +#if XSIMD_WITH_LASX +#include +#endif + +namespace xsimd +{ + /** + * @ingroup architectures + * + * Loongson Advanced SIMD Extension (LASX). + */ + struct lasx : common + { + static constexpr bool supported() noexcept { return XSIMD_WITH_LASX; } + static constexpr bool available() noexcept { return true; } + static constexpr bool requires_alignment() noexcept { return true; } + static constexpr std::size_t alignment() noexcept { return 32; } + static constexpr char const* name() noexcept { return "loongarch64+lasx"; } + }; + +#if XSIMD_WITH_LASX + namespace types + { + XSIMD_DECLARE_SIMD_REGISTER(signed char, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned char, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(char, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(short, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned short, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(int, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned int, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(long, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned long, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(long long, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned long long, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(float, lasx, __m256); + XSIMD_DECLARE_SIMD_REGISTER(double, lasx, __m256d); + + template + struct get_bool_simd_register + { + using type = simd_register, lasx>; + }; + } +#endif +} + +#endif diff --git a/include/xsimd/types/xsimd_lsx_register.hpp b/include/xsimd/types/xsimd_lsx_register.hpp index f8d2f66ce..c6abac40c 100644 --- a/include/xsimd/types/xsimd_lsx_register.hpp +++ b/include/xsimd/types/xsimd_lsx_register.hpp @@ -19,18 +19,18 @@ #include +#if XSIMD_WITH_LSX +#include +#endif + namespace xsimd { - struct loongarch : common - { - }; - /** * @ingroup architectures * - * LoongArch 128-bit SIMD extension. + * Loongson SIMD Extension (LSX). */ - struct lsx : loongarch + struct lsx : common { static constexpr bool supported() noexcept { return XSIMD_WITH_LSX; } static constexpr bool available() noexcept { return true; } @@ -39,109 +39,28 @@ namespace xsimd static constexpr char const* name() noexcept { return "loongarch64+lsx"; } }; - /** - * @ingroup architectures - * - * LoongArch 256-bit advanced SIMD extension. - */ - struct lasx : lsx - { - static constexpr bool supported() noexcept { return XSIMD_WITH_LASX; } - static constexpr bool available() noexcept { return true; } - static constexpr bool requires_alignment() noexcept { return true; } - static constexpr std::size_t alignment() noexcept { return 32; } - static constexpr char const* name() noexcept { return "loongarch64+lasx"; } - }; - -#if XSIMD_WITH_LSX || XSIMD_WITH_LASX +#if XSIMD_WITH_LSX namespace types { - namespace detail - { - template - struct loongarch_vector_type; - -#define XSIMD_DECLARE_LOONGARCH_VECTOR_TYPE(T, BYTES) \ - template <> \ - struct loongarch_vector_type \ - { \ - typedef T type __attribute__((vector_size(BYTES))); \ - } - -#define XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(T) \ - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPE(T, 16); \ - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPE(T, 32) + XSIMD_DECLARE_SIMD_REGISTER(signed char, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned char, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(char, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(short, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned short, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(int, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned int, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(long, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned long, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(long long, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned long long, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(float, lsx, __m128); + XSIMD_DECLARE_SIMD_REGISTER(double, lsx, __m128d); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(signed char); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(unsigned char); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(char); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(short); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(unsigned short); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(int); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(unsigned int); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(long); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(unsigned long); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(long long); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(unsigned long long); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(float); - XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES(double); - -#undef XSIMD_DECLARE_LOONGARCH_VECTOR_TYPES -#undef XSIMD_DECLARE_LOONGARCH_VECTOR_TYPE - - template - using loongarch_vector_type_t = typename loongarch_vector_type::type; - - template - using lsx_vector_type_t = loongarch_vector_type_t; - - template - using lasx_vector_type_t = loongarch_vector_type_t; - } - -#define XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(T, ARCH) \ - XSIMD_DECLARE_SIMD_REGISTER(T, ARCH, detail::ARCH##_vector_type_t) - -#define XSIMD_DECLARE_LOONGARCH_SIMD_REGISTERS(ARCH) \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(signed char, ARCH); \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(unsigned char, ARCH); \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(char, ARCH); \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(short, ARCH); \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(unsigned short, ARCH); \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(int, ARCH); \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(unsigned int, ARCH); \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(long, ARCH); \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(unsigned long, ARCH); \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(long long, ARCH); \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(unsigned long long, ARCH); \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(float, ARCH); \ - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER(double, ARCH) - -#if XSIMD_WITH_LSX - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTERS(lsx); -#endif -#if XSIMD_WITH_LASX - XSIMD_DECLARE_LOONGARCH_SIMD_REGISTERS(lasx); -#endif - -#undef XSIMD_DECLARE_LOONGARCH_SIMD_REGISTERS -#undef XSIMD_DECLARE_LOONGARCH_SIMD_REGISTER - -#if XSIMD_WITH_LSX template struct get_bool_simd_register { using type = simd_register, lsx>; }; -#endif - -#if XSIMD_WITH_LASX - template - struct get_bool_simd_register - { - using type = simd_register, lasx>; - }; -#endif } #endif } diff --git a/test/test_batch_cast.cpp b/test/test_batch_cast.cpp index 954ae0e66..81884948e 100644 --- a/test/test_batch_cast.cpp +++ b/test/test_batch_cast.cpp @@ -396,7 +396,7 @@ TYPED_TEST(batch_cast_test, cast_sizeshift2) } #endif -#if XSIMD_WITH_SSE2 +#if XSIMD_WITH_SSE2 || XSIMD_WITH_LSX || XSIMD_WITH_LASX TEST_CASE_TEMPLATE("[xsimd cast tests]", B, CONVERSION_TYPES) { SUBCASE("use fastcast")