diff --git a/.github/toolchains/gcc-loongarch64-linux-gnu.cmake b/.github/toolchains/gcc-loongarch64-linux-gnu.cmake new file mode 100644 index 000000000..472b4efb2 --- /dev/null +++ b/.github/toolchains/gcc-loongarch64-linux-gnu.cmake @@ -0,0 +1,4 @@ +set(CMAKE_SYSTEM_PROCESSOR loongarch64) +set(triple loongarch64-linux-gnu) + +include(${CMAKE_CURRENT_LIST_DIR}/gcc.cmake) diff --git a/.github/workflows/cross-loongarch.yml b/.github/workflows/cross-loongarch.yml new file mode 100644 index 000000000..8840fdd86 --- /dev/null +++ b/.github/workflows/cross-loongarch.yml @@ -0,0 +1,65 @@ +name: LoongArch64 cross-compilation build +on: [push, pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.job }}-${{ github.ref }} + cancel-in-progress: true +jobs: + build: + runs-on: ubuntu-latest + name: 'LoongArch64 ${{ matrix.simd.name }}' + strategy: + fail-fast: false + matrix: + simd: + - { name: 'LSX', flags: '-mlsx -mno-lasx', lasx: 0, cpu: 'la464,lsx=on,lasx=off' } + - { name: 'LASX', flags: '-mlasx', lasx: 1, cpu: 'la464,lsx=on,lasx=on' } + steps: + - name: Setup compiler + run: | + sudo apt-get -y -qq update + sudo apt-get -y -qq --no-install-suggests --no-install-recommends install \ + gcc-14-loongarch64-linux-gnu \ + g++-14-loongarch64-linux-gnu + sudo update-alternatives --install /usr/bin/loongarch64-linux-gnu-gcc \ + loongarch64-linux-gnu-gcc /usr/bin/loongarch64-linux-gnu-gcc-14 20 + sudo update-alternatives --install /usr/bin/loongarch64-linux-gnu-g++ \ + loongarch64-linux-gnu-g++ /usr/bin/loongarch64-linux-gnu-g++-14 20 + loongarch64-linux-gnu-g++ --version + - name: Setup QEMU + run: | + sudo apt-get -y -qq update + sudo apt-get -y -qq --no-install-suggests --no-install-recommends install qemu-user-static + qemu-loongarch64-static --version + - name: Setup Ninja + run: | + sudo apt-get -y -qq install ninja-build + - name: Checkout xsimd + uses: actions/checkout@v6 + - name: Setup + run: >- + cmake -S . -B _build + -GNinja + -DBUILD_TESTS=ON + -DDOWNLOAD_DOCTEST=ON + -DCMAKE_BUILD_TYPE=Release + -DTARGET_ARCH=generic + -DCMAKE_C_FLAGS="${{ matrix.simd.flags }}" + -DCMAKE_CXX_FLAGS="${{ matrix.simd.flags }}" + -DCMAKE_TOOLCHAIN_FILE=${{ github.workspace }}/.github/toolchains/gcc-loongarch64-linux-gnu.cmake + - name: Build + run: cmake --build _build + - name: Set CPU feature test expectations + run: | + echo "XSIMD_TEST_CPU_ASSUME_SSE4_2=0" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_NEON64=0" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_SVE=0" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_RVV=0" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_LSX=1" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_LASX=${{ matrix.simd.lasx }}" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_VSX=0" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_VXE=0" >> "$GITHUB_ENV" + echo "XSIMD_TEST_CPU_ASSUME_MANUFACTURER=unknown" >> "$GITHUB_ENV" + - name: Testing xsimd + timeout-minutes: 15 + run: qemu-loongarch64-static -cpu '${{ matrix.simd.cpu }}' -L /usr/loongarch64-linux-gnu ./test/test_xsimd + working-directory: ${{ github.workspace }}/_build diff --git a/README.md b/README.md index 894cf098d..05894a6b4 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ ARM | NEON, NEON64, SVE128/256/512 (fixed vector size) WebAssembly | WASM powerpc64 | VSX RISC-V | RISC-V128/256/512 (fixed vector size) +LoongArch64 | LSX, LASX IBM Z (s390x)| VXE (IBM z14) ## Installation diff --git a/docs/Doxyfile b/docs/Doxyfile index 6ea1a5664..1c5e5b6db 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -24,6 +24,8 @@ INPUT = ../include/xsimd/types/xsimd_api.hpp \ ../include/xsimd/types/xsimd_neon64_register.hpp \ ../include/xsimd/types/xsimd_neon_register.hpp \ ../include/xsimd/types/xsimd_rvv_register.hpp \ + ../include/xsimd/types/xsimd_lasx_register.hpp \ + ../include/xsimd/types/xsimd_lsx_register.hpp \ ../include/xsimd/types/xsimd_sse2_register.hpp \ ../include/xsimd/types/xsimd_sse3_register.hpp \ ../include/xsimd/types/xsimd_sse4_1_register.hpp \ diff --git a/include/xsimd/arch/xsimd_isa.hpp b/include/xsimd/arch/xsimd_isa.hpp index 87aaa6c4a..8077a8a49 100644 --- a/include/xsimd/arch/xsimd_isa.hpp +++ b/include/xsimd/arch/xsimd_isa.hpp @@ -154,6 +154,14 @@ #include "./xsimd_rvv.hpp" #endif +#if XSIMD_WITH_LSX +#include "./xsimd_lsx.hpp" +#endif + +#if XSIMD_WITH_LASX +#include "./xsimd_lasx.hpp" +#endif + #if XSIMD_WITH_WASM #include "./xsimd_wasm.hpp" #endif diff --git a/include/xsimd/arch/xsimd_lasx.hpp b/include/xsimd/arch/xsimd_lasx.hpp new file mode 100644 index 000000000..e5c92a319 --- /dev/null +++ b/include/xsimd/arch/xsimd_lasx.hpp @@ -0,0 +1,1012 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_LASX_HPP +#define XSIMD_LASX_HPP + +#include "../types/xsimd_batch_constant.hpp" +#include "../types/xsimd_lasx_register.hpp" +#include "../types/xsimd_utils.hpp" + +#include +#include +#include +#include + +namespace xsimd +{ + namespace kernel + { + namespace detail + { + template + struct lasx_set_vector + { + typedef T type __attribute__((vector_size(32))); + }; + + template + XSIMD_INLINE __m256i lasx_to_int(batch const& value) noexcept + { + return bit_cast<__m256i>(value.data); + } + + template + XSIMD_INLINE typename batch::register_type lasx_from_int(__m256i value) noexcept + { + return bit_cast::register_type>(value); + } + + // LASX mask extraction is 128-bit lane-local. Reorder or merge + // both halves so bit i always represents logical lane i. + template + XSIMD_INLINE std::uint32_t lasx_mask(__m256i value) noexcept + { + if constexpr (sizeof(T) == 1) + { + const __m256i sign_bits = __lasx_xvmskltz_b(value); + const auto lo = static_cast(__lasx_xvpickve2gr_w(sign_bits, 0)); + const auto hi = static_cast(__lasx_xvpickve2gr_w(sign_bits, 4)); + return lo | (hi << 16); + } + else if constexpr (sizeof(T) == 2) + { + const __m256i odd = __lasx_xvpickod_b(value, value); + const __m256i shuffled = __lasx_xvpermi_d(odd, 0xd8); + return static_cast(__lasx_xvpickve2gr_w(__lasx_xvmskltz_b(shuffled), 0)); + } + else if constexpr (sizeof(T) == 4) + { + const __m256i odd = __lasx_xvpickod_h(value, value); + const __m256i shuffled = __lasx_xvpermi_d(odd, 0xd8); + return static_cast(__lasx_xvpickve2gr_w(__lasx_xvmskltz_h(shuffled), 0)); + } + else + { + const __m256i odd = __lasx_xvpickod_w(value, value); + const __m256i shuffled = __lasx_xvpermi_d(odd, 0xd8); + return static_cast(__lasx_xvpickve2gr_w(__lasx_xvmskltz_w(shuffled), 0)); + } + } + + XSIMD_INLINE __m256 lasx_hadd_pair(__m256 lhs, __m256 rhs) noexcept + { + const __m256i even = __lasx_xvpickev_w(bit_cast<__m256i>(rhs), bit_cast<__m256i>(lhs)); + const __m256i odd = __lasx_xvpickod_w(bit_cast<__m256i>(rhs), bit_cast<__m256i>(lhs)); + return __lasx_xvfadd_s(bit_cast<__m256>(even), bit_cast<__m256>(odd)); + } + + XSIMD_INLINE __m256d lasx_hadd_pair(__m256d lhs, __m256d rhs) noexcept + { + const __m256i even = __lasx_xvpickev_d(bit_cast<__m256i>(rhs), bit_cast<__m256i>(lhs)); + const __m256i odd = __lasx_xvpickod_d(bit_cast<__m256i>(rhs), bit_cast<__m256i>(lhs)); + return __lasx_xvfadd_d(bit_cast<__m256d>(even), bit_cast<__m256d>(odd)); + } + + // fast_cast + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvffint_s_w(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvffint_s_wu(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvffint_d_l(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvffint_d_lu(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvftintrz_w_s(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvftintrz_wu_s(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvftintrz_l_d(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lasx_xvftintrz_lu_d(self.data); + } + } + + // abs + template + XSIMD_INLINE batch abs(batch const& self, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvbitclri_w(detail::lasx_to_int(self), 31)); + } + + template + XSIMD_INLINE batch abs(batch const& self, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvbitclri_d(detail::lasx_to_int(self), 63)); + } + + // add + template >> + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvadd_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvadd_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvadd_w(self.data, other.data); + else + return __lasx_xvadd_d(self.data, other.data); + } + + template + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfadd_s(self.data, other.data); + } + + template + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfadd_d(self.data, other.data); + } + + // all/any + template + XSIMD_INLINE bool all(batch_bool const& self, requires_arch) noexcept + { + constexpr std::uint64_t all_bits = (std::uint64_t(1) << batch_bool::size) - 1; + return detail::lasx_mask(self.data) == all_bits; + } + + template + XSIMD_INLINE bool any(batch_bool const& self, requires_arch) noexcept + { + return detail::lasx_mask(self.data) != 0; + } + + // batch_bool_cast + template + XSIMD_INLINE batch_bool batch_bool_cast(batch_bool const& self, batch_bool const&, requires_arch) noexcept + { + return self.data; + } + + // bitwise operations + template + XSIMD_INLINE batch bitwise_and(batch const& self, batch const& other, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvand_v(detail::lasx_to_int(self), detail::lasx_to_int(other))); + } + + template + XSIMD_INLINE batch_bool bitwise_and(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lasx_xvand_v(self.data, other.data); + } + + template + XSIMD_INLINE batch bitwise_andnot(batch const& self, batch const& other, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvandn_v(detail::lasx_to_int(other), detail::lasx_to_int(self))); + } + + template + XSIMD_INLINE batch_bool bitwise_andnot(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lasx_xvandn_v(other.data, self.data); + } + + template + XSIMD_INLINE batch bitwise_not(batch const& self, requires_arch) noexcept + { + const __m256i bits = detail::lasx_to_int(self); + return detail::lasx_from_int(__lasx_xvnor_v(bits, bits)); + } + + template + XSIMD_INLINE batch_bool bitwise_not(batch_bool const& self, requires_arch) noexcept + { + return __lasx_xvnor_v(self.data, self.data); + } + + template + XSIMD_INLINE batch bitwise_or(batch const& self, batch const& other, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvor_v(detail::lasx_to_int(self), detail::lasx_to_int(other))); + } + + template + XSIMD_INLINE batch_bool bitwise_or(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lasx_xvor_v(self.data, other.data); + } + + template + XSIMD_INLINE batch bitwise_xor(batch const& self, batch const& other, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvxor_v(detail::lasx_to_int(self), detail::lasx_to_int(other))); + } + + template + XSIMD_INLINE batch_bool bitwise_xor(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lasx_xvxor_v(self.data, other.data); + } + + template + XSIMD_INLINE batch bitwise_cast(batch const& self, batch const&, requires_arch) noexcept + { + return bit_cast::register_type>(self.data); + } + + // shifts + template >> + XSIMD_INLINE batch bitwise_lshift(batch const& self, std::int32_t other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvsll_b(self.data, __lasx_xvreplgr2vr_b(other)); + else if constexpr (sizeof(T) == 2) + return __lasx_xvsll_h(self.data, __lasx_xvreplgr2vr_h(other)); + else if constexpr (sizeof(T) == 4) + return __lasx_xvsll_w(self.data, __lasx_xvreplgr2vr_w(other)); + else + return __lasx_xvsll_d(self.data, __lasx_xvreplgr2vr_d(other)); + } + + template >> + XSIMD_INLINE batch bitwise_rshift(batch const& self, std::int32_t other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lasx_xvsra_b(self.data, __lasx_xvreplgr2vr_b(other)); + else if constexpr (sizeof(T) == 2) + return __lasx_xvsra_h(self.data, __lasx_xvreplgr2vr_h(other)); + else if constexpr (sizeof(T) == 4) + return __lasx_xvsra_w(self.data, __lasx_xvreplgr2vr_w(other)); + else + return __lasx_xvsra_d(self.data, __lasx_xvreplgr2vr_d(other)); + } + else + { + if constexpr (sizeof(T) == 1) + return __lasx_xvsrl_b(self.data, __lasx_xvreplgr2vr_b(other)); + else if constexpr (sizeof(T) == 2) + return __lasx_xvsrl_h(self.data, __lasx_xvreplgr2vr_h(other)); + else if constexpr (sizeof(T) == 4) + return __lasx_xvsrl_w(self.data, __lasx_xvreplgr2vr_w(other)); + else + return __lasx_xvsrl_d(self.data, __lasx_xvreplgr2vr_d(other)); + } + } + + // div + template + XSIMD_INLINE batch div(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfdiv_s(self.data, other.data); + } + + template + XSIMD_INLINE batch div(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfdiv_d(self.data, other.data); + } + + // broadcast + template >> + XSIMD_INLINE batch broadcast(T value, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvreplgr2vr_b(static_cast(value)); + else if constexpr (sizeof(T) == 2) + return __lasx_xvreplgr2vr_h(static_cast(value)); + else if constexpr (sizeof(T) == 4) + return __lasx_xvreplgr2vr_w(bit_cast>(value)); + else + return __lasx_xvreplgr2vr_d(bit_cast>(value)); + } + + template + XSIMD_INLINE batch broadcast(float value, requires_arch) noexcept + { + const auto bits = bit_cast(value); + return bit_cast<__m256>(__lasx_xvreplgr2vr_w(bits)); + } + + template + XSIMD_INLINE batch broadcast(double value, requires_arch) noexcept + { + const auto bits = bit_cast(value); + return bit_cast<__m256d>(__lasx_xvreplgr2vr_d(static_cast(bits))); + } + + // comparisons + template >> + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvseq_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvseq_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvseq_w(self.data, other.data); + else + return __lasx_xvseq_d(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_ceq_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_ceq_d(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool eq(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvseq_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvseq_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvseq_w(self.data, other.data); + else + return __lasx_xvseq_d(self.data, other.data); + } + + template >> + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lasx_xvslt_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvslt_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvslt_w(self.data, other.data); + else + return __lasx_xvslt_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lasx_xvslt_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvslt_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvslt_wu(self.data, other.data); + else + return __lasx_xvslt_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_clt_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_clt_d(self.data, other.data); + } + + template >> + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lasx_xvsle_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvsle_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvsle_w(self.data, other.data); + else + return __lasx_xvsle_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lasx_xvsle_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvsle_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvsle_wu(self.data, other.data); + else + return __lasx_xvsle_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_cle_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_cle_d(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool gt(batch const& self, batch const& other, requires_arch) noexcept + { + return lt(other, self, lasx {}); + } + + template + XSIMD_INLINE batch_bool ge(batch const& self, batch const& other, requires_arch) noexcept + { + return le(other, self, lasx {}); + } + + template >> + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + { + const __m256i equal = eq(self, other, lasx {}).data; + return __lasx_xvnor_v(equal, equal); + } + + template + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_cune_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfcmp_cune_d(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool neq(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lasx_xvxor_v(self.data, other.data); + } + + // first + template >> + XSIMD_INLINE T first(batch const& self, requires_arch) noexcept + { + if constexpr (sizeof(T) <= 4) + { + const auto word = static_cast(__lasx_xvpickve2gr_wu(self.data, 0)); + using unsigned_type = sized_uint_t; + return bit_cast(static_cast(word)); + } + else if constexpr (std::is_signed_v) + return static_cast(__lasx_xvpickve2gr_d(self.data, 0)); + else + return static_cast(__lasx_xvpickve2gr_du(self.data, 0)); + } + + template + XSIMD_INLINE float first(batch const& self, requires_arch) noexcept + { + const auto bits = static_cast(__lasx_xvpickve2gr_w(detail::lasx_to_int(self), 0)); + return bit_cast(bits); + } + + template + XSIMD_INLINE double first(batch const& self, requires_arch) noexcept + { + const auto bits = static_cast(__lasx_xvpickve2gr_du(detail::lasx_to_int(self), 0)); + return bit_cast(bits); + } + + template + XSIMD_INLINE std::complex first(batch, A> const& self, requires_arch) noexcept + { + return { first(self.real(), lasx {}), first(self.imag(), lasx {}) }; + } + + // horizontal add of rows + template + XSIMD_INLINE batch haddp(batch const* row, requires_arch) noexcept + { + __m256 abcd = detail::lasx_hadd_pair( + detail::lasx_hadd_pair(row[0].data, row[1].data), + detail::lasx_hadd_pair(row[2].data, row[3].data)); + __m256 efgh = detail::lasx_hadd_pair( + detail::lasx_hadd_pair(row[4].data, row[5].data), + detail::lasx_hadd_pair(row[6].data, row[7].data)); + const __m256i lo = __lasx_xvpermi_q(bit_cast<__m256i>(efgh), bit_cast<__m256i>(abcd), 0x30); + const __m256i hi = __lasx_xvpermi_q(bit_cast<__m256i>(efgh), bit_cast<__m256i>(abcd), 0x21); + return __lasx_xvfadd_s(bit_cast<__m256>(lo), bit_cast<__m256>(hi)); + } + + template + XSIMD_INLINE batch haddp(batch const* row, requires_arch) noexcept + { + const __m256d ab = detail::lasx_hadd_pair(row[0].data, row[1].data); + const __m256d cd = detail::lasx_hadd_pair(row[2].data, row[3].data); + const __m256i lo = __lasx_xvpermi_q(bit_cast<__m256i>(cd), bit_cast<__m256i>(ab), 0x30); + const __m256i hi = __lasx_xvpermi_q(bit_cast<__m256i>(cd), bit_cast<__m256i>(ab), 0x21); + return __lasx_xvfadd_d(bit_cast<__m256d>(lo), bit_cast<__m256d>(hi)); + } + + // load + template >> + XSIMD_INLINE batch load_unaligned(T const* mem, convert, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvld(mem, 0)); + } + + template >> + XSIMD_INLINE batch load_aligned(T const* mem, convert, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvld(mem, 0)); + } + + // load/store complex helpers + namespace detail + { + template + XSIMD_INLINE batch, A> load_complex(batch const& first_chunk, batch const& second_chunk, requires_arch) noexcept + { + __m256i real; + __m256i imag; + if constexpr (sizeof(T) == 4) + { + real = __lasx_xvpickev_w(lasx_to_int(second_chunk), lasx_to_int(first_chunk)); + imag = __lasx_xvpickod_w(lasx_to_int(second_chunk), lasx_to_int(first_chunk)); + } + else + { + real = __lasx_xvpickev_d(lasx_to_int(second_chunk), lasx_to_int(first_chunk)); + imag = __lasx_xvpickod_d(lasx_to_int(second_chunk), lasx_to_int(first_chunk)); + } + // xvpick* operates independently on each 128-bit lane; restore + // the linear element order expected by batch>. + real = __lasx_xvpermi_d(real, 0xd8); + imag = __lasx_xvpermi_d(imag, 0xd8); + return { lasx_from_int(real), lasx_from_int(imag) }; + } + + template + XSIMD_INLINE batch complex_low(batch, A> const& self, requires_arch) noexcept + { + __m256i lo; + __m256i hi; + if constexpr (sizeof(T) == 4) + { + lo = __lasx_xvilvl_w(lasx_to_int(self.imag()), lasx_to_int(self.real())); + hi = __lasx_xvilvh_w(lasx_to_int(self.imag()), lasx_to_int(self.real())); + } + else + { + lo = __lasx_xvilvl_d(lasx_to_int(self.imag()), lasx_to_int(self.real())); + hi = __lasx_xvilvh_d(lasx_to_int(self.imag()), lasx_to_int(self.real())); + } + return lasx_from_int(__lasx_xvpermi_q(hi, lo, 0x20)); + } + + template + XSIMD_INLINE batch complex_high(batch, A> const& self, requires_arch) noexcept + { + __m256i lo; + __m256i hi; + if constexpr (sizeof(T) == 4) + { + lo = __lasx_xvilvl_w(lasx_to_int(self.imag()), lasx_to_int(self.real())); + hi = __lasx_xvilvh_w(lasx_to_int(self.imag()), lasx_to_int(self.real())); + } + else + { + lo = __lasx_xvilvl_d(lasx_to_int(self.imag()), lasx_to_int(self.real())); + hi = __lasx_xvilvh_d(lasx_to_int(self.imag()), lasx_to_int(self.real())); + } + return lasx_from_int(__lasx_xvpermi_q(hi, lo, 0x31)); + } + } + + // max/min + template >> + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lasx_xvmax_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvmax_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvmax_w(self.data, other.data); + else + return __lasx_xvmax_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lasx_xvmax_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvmax_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvmax_wu(self.data, other.data); + else + return __lasx_xvmax_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept + { + const __m256i cond = __lasx_xvfcmp_clt_s(self.data, other.data); + return detail::lasx_from_int(__lasx_xvbitsel_v(detail::lasx_to_int(self), detail::lasx_to_int(other), cond)); + } + + template + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept + { + const __m256i cond = __lasx_xvfcmp_clt_d(self.data, other.data); + return detail::lasx_from_int(__lasx_xvbitsel_v(detail::lasx_to_int(self), detail::lasx_to_int(other), cond)); + } + + template >> + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lasx_xvmin_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvmin_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvmin_w(self.data, other.data); + else + return __lasx_xvmin_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lasx_xvmin_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvmin_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvmin_wu(self.data, other.data); + else + return __lasx_xvmin_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + const __m256i cond = __lasx_xvfcmp_clt_s(other.data, self.data); + return detail::lasx_from_int(__lasx_xvbitsel_v(detail::lasx_to_int(self), detail::lasx_to_int(other), cond)); + } + + template + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + const __m256i cond = __lasx_xvfcmp_clt_d(other.data, self.data); + return detail::lasx_from_int(__lasx_xvbitsel_v(detail::lasx_to_int(self), detail::lasx_to_int(other), cond)); + } + + // mul/neg + template >> + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvmul_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvmul_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvmul_w(self.data, other.data); + else + return __lasx_xvmul_d(self.data, other.data); + } + + template + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfmul_s(self.data, other.data); + } + + template + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfmul_d(self.data, other.data); + } + + template >> + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvneg_b(self.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvneg_h(self.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvneg_w(self.data); + else + return __lasx_xvneg_d(self.data); + } + + template + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + const auto sign_bit = bit_cast(std::uint32_t(1) << 31); + const __m256i sign = __lasx_xvreplgr2vr_w(sign_bit); + return detail::lasx_from_int(__lasx_xvxor_v(detail::lasx_to_int(self), sign)); + } + + template + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + const auto sign_bit = bit_cast(std::uint64_t(1) << 63); + const __m256i sign = __lasx_xvreplgr2vr_d(sign_bit); + return detail::lasx_from_int(__lasx_xvxor_v(detail::lasx_to_int(self), sign)); + } + + // rsqrt/sqrt + template + XSIMD_INLINE batch rsqrt(batch const& self, requires_arch) noexcept + { + return __lasx_xvfrsqrt_s(self.data); + } + + template + XSIMD_INLINE batch rsqrt(batch const& self, requires_arch) noexcept + { + return __lasx_xvfrsqrt_d(self.data); + } + + template + XSIMD_INLINE batch sqrt(batch const& self, requires_arch) noexcept + { + return __lasx_xvfsqrt_s(self.data); + } + + template + XSIMD_INLINE batch sqrt(batch const& self, requires_arch) noexcept + { + return __lasx_xvfsqrt_d(self.data); + } + + // isnan + template + XSIMD_INLINE batch_bool isnan(batch const& self, requires_arch) noexcept + { + return __lasx_xvfcmp_cun_s(self.data, self.data); + } + + template + XSIMD_INLINE batch_bool isnan(batch const& self, requires_arch) noexcept + { + return __lasx_xvfcmp_cun_d(self.data, self.data); + } + + // select + template + XSIMD_INLINE batch select(batch_bool const& cond, batch const& true_br, batch const& false_br, requires_arch) noexcept + { + return detail::lasx_from_int(__lasx_xvbitsel_v(detail::lasx_to_int(false_br), detail::lasx_to_int(true_br), cond.data)); + } + + template + XSIMD_INLINE batch select(batch_bool_constant const&, batch const& true_br, batch const& false_br, requires_arch) noexcept + { + return select(batch_bool { Values... }, true_br, false_br, lasx {}); + } + + // set + template + XSIMD_INLINE batch set(batch const&, requires_arch, Values... values) noexcept + { + static_assert(sizeof...(Values) == batch::size, "consistent init"); + using vector_type = typename detail::lasx_set_vector::type; + const vector_type vector = { static_cast(values)... }; + return bit_cast::register_type>(vector); + } + + template + XSIMD_INLINE batch, A> set(batch, A> const&, requires_arch, Values... values) noexcept + { + return { set(batch {}, lasx {}, values.real()...), + set(batch {}, lasx {}, values.imag()...) }; + } + + template + XSIMD_INLINE batch_bool set(batch_bool const&, requires_arch, Values... values) noexcept + { + using value_type = sized_uint_t; + return set(batch {}, lasx {}, static_cast(values ? ~value_type(0) : value_type(0))...).data; + } + + // byte slides + // xvbsll/xvbsrl operate independently on each 128-bit lane, so the + // adjacent lane is permuted in to supply or receive the carried bytes. + template + XSIMD_INLINE batch slide_left(batch const& self, requires_arch) noexcept + { + static_assert(N <= 32, "invalid byte slide"); + const __m256i bits = detail::lasx_to_int(self); + const __m256i zero = __lasx_xvldi(0); + if constexpr (N == 0) + return self; + else if constexpr (N < 16) + { + const __m256i shifted = __lasx_xvbsll_v(bits, N); + const __m256i previous = __lasx_xvpermi_q(bits, zero, 0x20); + const __m256i carry = __lasx_xvbsrl_v(previous, 16 - N); + return detail::lasx_from_int(__lasx_xvor_v(shifted, carry)); + } + else if constexpr (N == 16) + return detail::lasx_from_int(__lasx_xvpermi_q(bits, zero, 0x20)); + else if constexpr (N < 32) + { + const __m256i previous = __lasx_xvpermi_q(bits, zero, 0x20); + return detail::lasx_from_int(__lasx_xvbsll_v(previous, N - 16)); + } + else + return detail::lasx_from_int(zero); + } + + template + XSIMD_INLINE batch slide_right(batch const& self, requires_arch) noexcept + { + static_assert(N <= 32, "invalid byte slide"); + const __m256i bits = detail::lasx_to_int(self); + const __m256i zero = __lasx_xvldi(0); + if constexpr (N == 0) + return self; + else if constexpr (N < 16) + { + const __m256i shifted = __lasx_xvbsrl_v(bits, N); + const __m256i next = __lasx_xvpermi_q(zero, bits, 0x31); + const __m256i carry = __lasx_xvbsll_v(next, 16 - N); + return detail::lasx_from_int(__lasx_xvor_v(shifted, carry)); + } + else if constexpr (N == 16) + return detail::lasx_from_int(__lasx_xvpermi_q(zero, bits, 0x31)); + else if constexpr (N < 32) + { + const __m256i next = __lasx_xvpermi_q(zero, bits, 0x31); + return detail::lasx_from_int(__lasx_xvbsrl_v(next, N - 16)); + } + else + return detail::lasx_from_int(zero); + } + + // store + template >> + XSIMD_INLINE void store_unaligned(T* mem, batch const& self, requires_arch) noexcept + { + __lasx_xvst(detail::lasx_to_int(self), mem, 0); + } + + template >> + XSIMD_INLINE void store_aligned(T* mem, batch const& self, requires_arch) noexcept + { + __lasx_xvst(detail::lasx_to_int(self), mem, 0); + } + + // sub + template >> + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lasx_xvsub_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lasx_xvsub_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lasx_xvsub_w(self.data, other.data); + else + return __lasx_xvsub_d(self.data, other.data); + } + + template + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfsub_s(self.data, other.data); + } + + template + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + { + return __lasx_xvfsub_d(self.data, other.data); + } + + // zip + // xvilvl/xvilvh interleave within 128-bit lanes; xvpermi_q assembles + // the full-width lower or upper half of the logical 256-bit vectors. + template + XSIMD_INLINE batch zip_lo(batch const& self, batch const& other, requires_arch) noexcept + { + __m256i lo; + __m256i hi; + if constexpr (sizeof(T) == 1) + { + lo = __lasx_xvilvl_b(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_b(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + else if constexpr (sizeof(T) == 2) + { + lo = __lasx_xvilvl_h(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_h(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + else if constexpr (sizeof(T) == 4) + { + lo = __lasx_xvilvl_w(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_w(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + else + { + lo = __lasx_xvilvl_d(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_d(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + return detail::lasx_from_int(__lasx_xvpermi_q(hi, lo, 0x20)); + } + + template + XSIMD_INLINE batch zip_hi(batch const& self, batch const& other, requires_arch) noexcept + { + __m256i lo; + __m256i hi; + if constexpr (sizeof(T) == 1) + { + lo = __lasx_xvilvl_b(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_b(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + else if constexpr (sizeof(T) == 2) + { + lo = __lasx_xvilvl_h(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_h(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + else if constexpr (sizeof(T) == 4) + { + lo = __lasx_xvilvl_w(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_w(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + else + { + lo = __lasx_xvilvl_d(detail::lasx_to_int(other), detail::lasx_to_int(self)); + hi = __lasx_xvilvh_d(detail::lasx_to_int(other), detail::lasx_to_int(self)); + } + return detail::lasx_from_int(__lasx_xvpermi_q(hi, lo, 0x31)); + } + } +} + +#endif diff --git a/include/xsimd/arch/xsimd_lsx.hpp b/include/xsimd/arch/xsimd_lsx.hpp new file mode 100644 index 000000000..af86b2b01 --- /dev/null +++ b/include/xsimd/arch/xsimd_lsx.hpp @@ -0,0 +1,915 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_LSX_HPP +#define XSIMD_LSX_HPP + +#include "../types/xsimd_batch_constant.hpp" +#include "../types/xsimd_lsx_register.hpp" +#include "../types/xsimd_utils.hpp" + +#include +#include +#include +#include + +namespace xsimd +{ + namespace kernel + { + namespace detail + { + template + struct lsx_set_vector + { + typedef T type __attribute__((vector_size(16))); + }; + + template + XSIMD_INLINE __m128i lsx_to_int(batch const& value) noexcept + { + return bit_cast<__m128i>(value.data); + } + + template + XSIMD_INLINE typename batch::register_type lsx_from_int(__m128i value) noexcept + { + return bit_cast::register_type>(value); + } + + template + XSIMD_INLINE std::uint32_t lsx_mask(__m128i value) noexcept + { + if constexpr (sizeof(T) == 1) + { + return static_cast(__lsx_vpickve2gr_w(__lsx_vmskltz_b(value), 0)); + } + else if constexpr (sizeof(T) == 2) + { + return static_cast(__lsx_vpickve2gr_w(__lsx_vmskltz_h(value), 0)); + } + else if constexpr (sizeof(T) == 4) + { + return static_cast(__lsx_vpickve2gr_w(__lsx_vmskltz_w(value), 0)); + } + else + { + return static_cast(__lsx_vpickve2gr_w(__lsx_vmskltz_d(value), 0)); + } + } + + // fast_cast + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vffint_s_w(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vffint_s_wu(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vffint_d_l(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vffint_d_lu(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vftintrz_w_s(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vftintrz_wu_s(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vftintrz_l_d(self.data); + } + + template + XSIMD_INLINE batch fast_cast(batch const& self, batch const&, requires_arch) noexcept + { + return __lsx_vftintrz_lu_d(self.data); + } + } + + // abs + template + XSIMD_INLINE batch abs(batch const& self, requires_arch) noexcept + { + return detail::lsx_from_int(__lsx_vbitclri_w(detail::lsx_to_int(self), 31)); + } + + template + XSIMD_INLINE batch abs(batch const& self, requires_arch) noexcept + { + return detail::lsx_from_int(__lsx_vbitclri_d(detail::lsx_to_int(self), 63)); + } + + // add + template >> + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lsx_vadd_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vadd_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vadd_w(self.data, other.data); + else + return __lsx_vadd_d(self.data, other.data); + } + + template + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfadd_s(self.data, other.data); + } + + template + XSIMD_INLINE batch add(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfadd_d(self.data, other.data); + } + + // all/any + template + XSIMD_INLINE bool all(batch_bool const& self, requires_arch) noexcept + { + constexpr std::uint32_t all_bits = (std::uint32_t(1) << batch_bool::size) - 1; + return detail::lsx_mask(self.data) == all_bits; + } + + template + XSIMD_INLINE bool any(batch_bool const& self, requires_arch) noexcept + { + return detail::lsx_mask(self.data) != 0; + } + + // batch_bool_cast + template + XSIMD_INLINE batch_bool batch_bool_cast(batch_bool const& self, batch_bool const&, requires_arch) noexcept + { + return self.data; + } + + // bitwise operations + template + XSIMD_INLINE batch bitwise_and(batch const& self, batch const& other, requires_arch) noexcept + { + return detail::lsx_from_int(__lsx_vand_v(detail::lsx_to_int(self), detail::lsx_to_int(other))); + } + + template + XSIMD_INLINE batch_bool bitwise_and(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lsx_vand_v(self.data, other.data); + } + + template + XSIMD_INLINE batch bitwise_andnot(batch const& self, batch const& other, requires_arch) noexcept + { + return detail::lsx_from_int(__lsx_vandn_v(detail::lsx_to_int(other), detail::lsx_to_int(self))); + } + + template + XSIMD_INLINE batch_bool bitwise_andnot(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lsx_vandn_v(other.data, self.data); + } + + template + XSIMD_INLINE batch bitwise_not(batch const& self, requires_arch) noexcept + { + const __m128i bits = detail::lsx_to_int(self); + return detail::lsx_from_int(__lsx_vnor_v(bits, bits)); + } + + template + XSIMD_INLINE batch_bool bitwise_not(batch_bool const& self, requires_arch) noexcept + { + return __lsx_vnor_v(self.data, self.data); + } + + template + XSIMD_INLINE batch bitwise_or(batch const& self, batch const& other, requires_arch) noexcept + { + return detail::lsx_from_int(__lsx_vor_v(detail::lsx_to_int(self), detail::lsx_to_int(other))); + } + + template + XSIMD_INLINE batch_bool bitwise_or(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lsx_vor_v(self.data, other.data); + } + + template + XSIMD_INLINE batch bitwise_xor(batch const& self, batch const& other, requires_arch) noexcept + { + return detail::lsx_from_int(__lsx_vxor_v(detail::lsx_to_int(self), detail::lsx_to_int(other))); + } + + template + XSIMD_INLINE batch_bool bitwise_xor(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lsx_vxor_v(self.data, other.data); + } + + template + XSIMD_INLINE batch bitwise_cast(batch const& self, batch const&, requires_arch) noexcept + { + return bit_cast::register_type>(self.data); + } + + // shifts + template >> + XSIMD_INLINE batch bitwise_lshift(batch const& self, std::int32_t other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lsx_vsll_b(self.data, __lsx_vreplgr2vr_b(other)); + else if constexpr (sizeof(T) == 2) + return __lsx_vsll_h(self.data, __lsx_vreplgr2vr_h(other)); + else if constexpr (sizeof(T) == 4) + return __lsx_vsll_w(self.data, __lsx_vreplgr2vr_w(other)); + else + return __lsx_vsll_d(self.data, __lsx_vreplgr2vr_d(other)); + } + + template >> + XSIMD_INLINE batch bitwise_rshift(batch const& self, std::int32_t other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lsx_vsra_b(self.data, __lsx_vreplgr2vr_b(other)); + else if constexpr (sizeof(T) == 2) + return __lsx_vsra_h(self.data, __lsx_vreplgr2vr_h(other)); + else if constexpr (sizeof(T) == 4) + return __lsx_vsra_w(self.data, __lsx_vreplgr2vr_w(other)); + else + return __lsx_vsra_d(self.data, __lsx_vreplgr2vr_d(other)); + } + else + { + if constexpr (sizeof(T) == 1) + return __lsx_vsrl_b(self.data, __lsx_vreplgr2vr_b(other)); + else if constexpr (sizeof(T) == 2) + return __lsx_vsrl_h(self.data, __lsx_vreplgr2vr_h(other)); + else if constexpr (sizeof(T) == 4) + return __lsx_vsrl_w(self.data, __lsx_vreplgr2vr_w(other)); + else + return __lsx_vsrl_d(self.data, __lsx_vreplgr2vr_d(other)); + } + } + + // div + template + XSIMD_INLINE batch div(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfdiv_s(self.data, other.data); + } + + template + XSIMD_INLINE batch div(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfdiv_d(self.data, other.data); + } + + // broadcast + template >> + XSIMD_INLINE batch broadcast(T value, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lsx_vreplgr2vr_b(static_cast(value)); + else if constexpr (sizeof(T) == 2) + return __lsx_vreplgr2vr_h(static_cast(value)); + else if constexpr (sizeof(T) == 4) + return __lsx_vreplgr2vr_w(bit_cast>(value)); + else + return __lsx_vreplgr2vr_d(bit_cast>(value)); + } + + template + XSIMD_INLINE batch broadcast(float value, requires_arch) noexcept + { + const auto bits = bit_cast(value); + return bit_cast<__m128>(__lsx_vreplgr2vr_w(bits)); + } + + template + XSIMD_INLINE batch broadcast(double value, requires_arch) noexcept + { + const auto bits = bit_cast(value); + return bit_cast<__m128d>(__lsx_vreplgr2vr_d(static_cast(bits))); + } + + // comparisons + template >> + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lsx_vseq_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vseq_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vseq_w(self.data, other.data); + else + return __lsx_vseq_d(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfcmp_ceq_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool eq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfcmp_ceq_d(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool eq(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lsx_vseq_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vseq_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vseq_w(self.data, other.data); + else + return __lsx_vseq_d(self.data, other.data); + } + + template >> + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lsx_vslt_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vslt_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vslt_w(self.data, other.data); + else + return __lsx_vslt_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lsx_vslt_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vslt_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vslt_wu(self.data, other.data); + else + return __lsx_vslt_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfcmp_clt_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool lt(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfcmp_clt_d(self.data, other.data); + } + + template >> + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lsx_vsle_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vsle_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vsle_w(self.data, other.data); + else + return __lsx_vsle_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lsx_vsle_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vsle_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vsle_wu(self.data, other.data); + else + return __lsx_vsle_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfcmp_cle_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfcmp_cle_d(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool gt(batch const& self, batch const& other, requires_arch) noexcept + { + return lt(other, self, lsx {}); + } + + template + XSIMD_INLINE batch_bool ge(batch const& self, batch const& other, requires_arch) noexcept + { + return le(other, self, lsx {}); + } + + template >> + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + { + const __m128i equal = eq(self, other, lsx {}).data; + return __lsx_vnor_v(equal, equal); + } + + template + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfcmp_cune_s(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool neq(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfcmp_cune_d(self.data, other.data); + } + + template + XSIMD_INLINE batch_bool neq(batch_bool const& self, batch_bool const& other, requires_arch) noexcept + { + return __lsx_vxor_v(self.data, other.data); + } + + // first + template >> + XSIMD_INLINE T first(batch const& self, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_signed_v) + return static_cast(__lsx_vpickve2gr_b(self.data, 0)); + else + return static_cast(__lsx_vpickve2gr_bu(self.data, 0)); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_signed_v) + return static_cast(__lsx_vpickve2gr_h(self.data, 0)); + else + return static_cast(__lsx_vpickve2gr_hu(self.data, 0)); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_signed_v) + return static_cast(__lsx_vpickve2gr_w(self.data, 0)); + else + return static_cast(__lsx_vpickve2gr_wu(self.data, 0)); + } + else + { + if constexpr (std::is_signed_v) + return static_cast(__lsx_vpickve2gr_d(self.data, 0)); + else + return static_cast(__lsx_vpickve2gr_du(self.data, 0)); + } + } + + template + XSIMD_INLINE float first(batch const& self, requires_arch) noexcept + { + const auto bits = static_cast(__lsx_vpickve2gr_w(detail::lsx_to_int(self), 0)); + return bit_cast(bits); + } + + template + XSIMD_INLINE double first(batch const& self, requires_arch) noexcept + { + const auto bits = static_cast(__lsx_vpickve2gr_du(detail::lsx_to_int(self), 0)); + return bit_cast(bits); + } + + template + XSIMD_INLINE std::complex first(batch, A> const& self, requires_arch) noexcept + { + return { first(self.real(), lsx {}), first(self.imag(), lsx {}) }; + } + + // horizontal add of rows + template + XSIMD_INLINE batch haddp(batch const* row, requires_arch) noexcept + { + const __m128i ab_lo = __lsx_vilvl_w(detail::lsx_to_int(row[1]), detail::lsx_to_int(row[0])); + const __m128i ab_hi = __lsx_vilvh_w(detail::lsx_to_int(row[1]), detail::lsx_to_int(row[0])); + const __m128 ab = __lsx_vfadd_s(bit_cast<__m128>(ab_lo), bit_cast<__m128>(ab_hi)); + const __m128i cd_lo = __lsx_vilvl_w(detail::lsx_to_int(row[3]), detail::lsx_to_int(row[2])); + const __m128i cd_hi = __lsx_vilvh_w(detail::lsx_to_int(row[3]), detail::lsx_to_int(row[2])); + const __m128 cd = __lsx_vfadd_s(bit_cast<__m128>(cd_lo), bit_cast<__m128>(cd_hi)); + const __m128i lo = __lsx_vilvl_d(bit_cast<__m128i>(cd), bit_cast<__m128i>(ab)); + const __m128i hi = __lsx_vilvh_d(bit_cast<__m128i>(cd), bit_cast<__m128i>(ab)); + return __lsx_vfadd_s(bit_cast<__m128>(lo), bit_cast<__m128>(hi)); + } + + template + XSIMD_INLINE batch haddp(batch const* row, requires_arch) noexcept + { + const __m128i lo = __lsx_vilvl_d(detail::lsx_to_int(row[1]), detail::lsx_to_int(row[0])); + const __m128i hi = __lsx_vilvh_d(detail::lsx_to_int(row[1]), detail::lsx_to_int(row[0])); + return __lsx_vfadd_d(bit_cast<__m128d>(lo), bit_cast<__m128d>(hi)); + } + + // load + template >> + XSIMD_INLINE batch load_unaligned(T const* mem, convert, requires_arch) noexcept + { + return detail::lsx_from_int(__lsx_vld(mem, 0)); + } + + template >> + XSIMD_INLINE batch load_aligned(T const* mem, convert, requires_arch) noexcept + { + return detail::lsx_from_int(__lsx_vld(mem, 0)); + } + + // load/store complex helpers + namespace detail + { + template + XSIMD_INLINE batch, A> load_complex(batch const& first_chunk, batch const& second_chunk, requires_arch) noexcept + { + __m128i real; + __m128i imag; + if constexpr (sizeof(T) == 4) + { + real = __lsx_vpickev_w(lsx_to_int(second_chunk), lsx_to_int(first_chunk)); + imag = __lsx_vpickod_w(lsx_to_int(second_chunk), lsx_to_int(first_chunk)); + } + else + { + real = __lsx_vpickev_d(lsx_to_int(second_chunk), lsx_to_int(first_chunk)); + imag = __lsx_vpickod_d(lsx_to_int(second_chunk), lsx_to_int(first_chunk)); + } + return { lsx_from_int(real), lsx_from_int(imag) }; + } + + template + XSIMD_INLINE batch complex_low(batch, A> const& self, requires_arch) noexcept + { + if constexpr (sizeof(T) == 4) + return lsx_from_int(__lsx_vilvl_w(lsx_to_int(self.imag()), lsx_to_int(self.real()))); + else + return lsx_from_int(__lsx_vilvl_d(lsx_to_int(self.imag()), lsx_to_int(self.real()))); + } + + template + XSIMD_INLINE batch complex_high(batch, A> const& self, requires_arch) noexcept + { + if constexpr (sizeof(T) == 4) + return lsx_from_int(__lsx_vilvh_w(lsx_to_int(self.imag()), lsx_to_int(self.real()))); + else + return lsx_from_int(__lsx_vilvh_d(lsx_to_int(self.imag()), lsx_to_int(self.real()))); + } + } + + // max/min + template >> + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lsx_vmax_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vmax_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vmax_w(self.data, other.data); + else + return __lsx_vmax_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lsx_vmax_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vmax_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vmax_wu(self.data, other.data); + else + return __lsx_vmax_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept + { + const __m128i cond = __lsx_vfcmp_clt_s(self.data, other.data); + return detail::lsx_from_int(__lsx_vbitsel_v(detail::lsx_to_int(self), detail::lsx_to_int(other), cond)); + } + + template + XSIMD_INLINE batch max(batch const& self, batch const& other, requires_arch) noexcept + { + const __m128i cond = __lsx_vfcmp_clt_d(self.data, other.data); + return detail::lsx_from_int(__lsx_vbitsel_v(detail::lsx_to_int(self), detail::lsx_to_int(other), cond)); + } + + template >> + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (std::is_signed_v) + { + if constexpr (sizeof(T) == 1) + return __lsx_vmin_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vmin_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vmin_w(self.data, other.data); + else + return __lsx_vmin_d(self.data, other.data); + } + else + { + if constexpr (sizeof(T) == 1) + return __lsx_vmin_bu(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vmin_hu(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vmin_wu(self.data, other.data); + else + return __lsx_vmin_du(self.data, other.data); + } + } + + template + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + const __m128i cond = __lsx_vfcmp_clt_s(other.data, self.data); + return detail::lsx_from_int(__lsx_vbitsel_v(detail::lsx_to_int(self), detail::lsx_to_int(other), cond)); + } + + template + XSIMD_INLINE batch min(batch const& self, batch const& other, requires_arch) noexcept + { + const __m128i cond = __lsx_vfcmp_clt_d(other.data, self.data); + return detail::lsx_from_int(__lsx_vbitsel_v(detail::lsx_to_int(self), detail::lsx_to_int(other), cond)); + } + + // mul/neg + template >> + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lsx_vmul_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vmul_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vmul_w(self.data, other.data); + else + return __lsx_vmul_d(self.data, other.data); + } + + template + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfmul_s(self.data, other.data); + } + + template + XSIMD_INLINE batch mul(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfmul_d(self.data, other.data); + } + + template >> + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lsx_vneg_b(self.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vneg_h(self.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vneg_w(self.data); + else + return __lsx_vneg_d(self.data); + } + + template + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + const auto sign_bit = bit_cast(std::uint32_t(1) << 31); + const __m128i sign = __lsx_vreplgr2vr_w(sign_bit); + return detail::lsx_from_int(__lsx_vxor_v(detail::lsx_to_int(self), sign)); + } + + template + XSIMD_INLINE batch neg(batch const& self, requires_arch) noexcept + { + const auto sign_bit = bit_cast(std::uint64_t(1) << 63); + const __m128i sign = __lsx_vreplgr2vr_d(sign_bit); + return detail::lsx_from_int(__lsx_vxor_v(detail::lsx_to_int(self), sign)); + } + + // rsqrt/sqrt + template + XSIMD_INLINE batch rsqrt(batch const& self, requires_arch) noexcept + { + return __lsx_vfrsqrt_s(self.data); + } + + template + XSIMD_INLINE batch rsqrt(batch const& self, requires_arch) noexcept + { + return __lsx_vfrsqrt_d(self.data); + } + + template + XSIMD_INLINE batch sqrt(batch const& self, requires_arch) noexcept + { + return __lsx_vfsqrt_s(self.data); + } + + template + XSIMD_INLINE batch sqrt(batch const& self, requires_arch) noexcept + { + return __lsx_vfsqrt_d(self.data); + } + + // isnan + template + XSIMD_INLINE batch_bool isnan(batch const& self, requires_arch) noexcept + { + return __lsx_vfcmp_cun_s(self.data, self.data); + } + + template + XSIMD_INLINE batch_bool isnan(batch const& self, requires_arch) noexcept + { + return __lsx_vfcmp_cun_d(self.data, self.data); + } + + // select + template + XSIMD_INLINE batch select(batch_bool const& cond, batch const& true_br, batch const& false_br, requires_arch) noexcept + { + return detail::lsx_from_int(__lsx_vbitsel_v(detail::lsx_to_int(false_br), detail::lsx_to_int(true_br), cond.data)); + } + + template + XSIMD_INLINE batch select(batch_bool_constant const&, batch const& true_br, batch const& false_br, requires_arch) noexcept + { + return select(batch_bool { Values... }, true_br, false_br, lsx {}); + } + + // set + template + XSIMD_INLINE batch set(batch const&, requires_arch, Values... values) noexcept + { + static_assert(sizeof...(Values) == batch::size, "consistent init"); + using vector_type = typename detail::lsx_set_vector::type; + const vector_type vector = { static_cast(values)... }; + return bit_cast::register_type>(vector); + } + + template + XSIMD_INLINE batch, A> set(batch, A> const&, requires_arch, Values... values) noexcept + { + return { set(batch {}, lsx {}, values.real()...), + set(batch {}, lsx {}, values.imag()...) }; + } + + template + XSIMD_INLINE batch_bool set(batch_bool const&, requires_arch, Values... values) noexcept + { + using value_type = sized_uint_t; + return set(batch {}, lsx {}, static_cast(values ? ~value_type(0) : value_type(0))...).data; + } + + // byte slides + template + XSIMD_INLINE batch slide_left(batch const& self, requires_arch) noexcept + { + static_assert(N <= 16, "invalid byte slide"); + if constexpr (N == 16) + return detail::lsx_from_int(__lsx_vldi(0)); + else + return detail::lsx_from_int(__lsx_vbsll_v(detail::lsx_to_int(self), N)); + } + + template + XSIMD_INLINE batch slide_right(batch const& self, requires_arch) noexcept + { + static_assert(N <= 16, "invalid byte slide"); + if constexpr (N == 16) + return detail::lsx_from_int(__lsx_vldi(0)); + else + return detail::lsx_from_int(__lsx_vbsrl_v(detail::lsx_to_int(self), N)); + } + + // store + template >> + XSIMD_INLINE void store_unaligned(T* mem, batch const& self, requires_arch) noexcept + { + __lsx_vst(detail::lsx_to_int(self), mem, 0); + } + + template >> + XSIMD_INLINE void store_aligned(T* mem, batch const& self, requires_arch) noexcept + { + __lsx_vst(detail::lsx_to_int(self), mem, 0); + } + + // sub + template >> + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return __lsx_vsub_b(self.data, other.data); + else if constexpr (sizeof(T) == 2) + return __lsx_vsub_h(self.data, other.data); + else if constexpr (sizeof(T) == 4) + return __lsx_vsub_w(self.data, other.data); + else + return __lsx_vsub_d(self.data, other.data); + } + + template + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfsub_s(self.data, other.data); + } + + template + XSIMD_INLINE batch sub(batch const& self, batch const& other, requires_arch) noexcept + { + return __lsx_vfsub_d(self.data, other.data); + } + + // zip + template + XSIMD_INLINE batch zip_lo(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return detail::lsx_from_int(__lsx_vilvl_b(detail::lsx_to_int(other), detail::lsx_to_int(self))); + else if constexpr (sizeof(T) == 2) + return detail::lsx_from_int(__lsx_vilvl_h(detail::lsx_to_int(other), detail::lsx_to_int(self))); + else if constexpr (sizeof(T) == 4) + return detail::lsx_from_int(__lsx_vilvl_w(detail::lsx_to_int(other), detail::lsx_to_int(self))); + else + return detail::lsx_from_int(__lsx_vilvl_d(detail::lsx_to_int(other), detail::lsx_to_int(self))); + } + + template + XSIMD_INLINE batch zip_hi(batch const& self, batch const& other, requires_arch) noexcept + { + if constexpr (sizeof(T) == 1) + return detail::lsx_from_int(__lsx_vilvh_b(detail::lsx_to_int(other), detail::lsx_to_int(self))); + else if constexpr (sizeof(T) == 2) + return detail::lsx_from_int(__lsx_vilvh_h(detail::lsx_to_int(other), detail::lsx_to_int(self))); + else if constexpr (sizeof(T) == 4) + return detail::lsx_from_int(__lsx_vilvh_w(detail::lsx_to_int(other), detail::lsx_to_int(self))); + else + return detail::lsx_from_int(__lsx_vilvh_d(detail::lsx_to_int(other), detail::lsx_to_int(self))); + } + } +} + +#endif diff --git a/include/xsimd/config/xsimd_arch.hpp b/include/xsimd/config/xsimd_arch.hpp index ac5546bbc..a84d70d84 100644 --- a/include/xsimd/config/xsimd_arch.hpp +++ b/include/xsimd/config/xsimd_arch.hpp @@ -154,9 +154,10 @@ namespace xsimd using all_arm_architectures = typename detail::join, neon64, neon>>::type; using all_power_architectures = arch_list; using all_riscv_architectures = all_rvv_architectures; + using all_loongarch_architectures = arch_list; using all_wasm_architectures = arch_list; using all_s390x_architectures = arch_list; - using all_architectures = typename detail::join::type; + using all_architectures = typename detail::join::type; using supported_architectures = typename detail::supported::type; @@ -164,6 +165,7 @@ namespace xsimd using arm_arch = typename detail::supported::type::best; using power_arch = typename detail::supported::type::best; using riscv_arch = typename detail::supported::type::best; + using loongarch_arch = typename detail::supported::type::best; using s390x_arch = typename detail::supported::type::best; using best_arch = typename supported_architectures::best; diff --git a/include/xsimd/config/xsimd_config.hpp b/include/xsimd/config/xsimd_config.hpp index 4bd2a341b..be6fada32 100644 --- a/include/xsimd/config/xsimd_config.hpp +++ b/include/xsimd/config/xsimd_config.hpp @@ -512,6 +512,39 @@ #define XSIMD_RVV_BITS 0 #endif +/** + * @ingroup xsimd_config_macro + * + * Set to 1 if the target is the 64-bit LoongArch architecture family. + */ +#if defined(__loongarch64) || defined(__loongarch64__) || (defined(__loongarch_grlen) && __loongarch_grlen == 64) +#define XSIMD_TARGET_LOONGARCH64 1 +#else +#define XSIMD_TARGET_LOONGARCH64 0 +#endif + +/** + * @ingroup xsimd_config_macro + * + * Set to 1 if the 128-bit Loongson SIMD Extension is available. + */ +#if XSIMD_TARGET_LOONGARCH64 && (defined(__loongarch_sx) || defined(__loongarch_asx)) +#define XSIMD_WITH_LSX 1 +#else +#define XSIMD_WITH_LSX 0 +#endif + +/** + * @ingroup xsimd_config_macro + * + * Set to 1 if the 256-bit Loongson Advanced SIMD Extension is available. + */ +#if XSIMD_TARGET_LOONGARCH64 && defined(__loongarch_asx) +#define XSIMD_WITH_LASX 1 +#else +#define XSIMD_WITH_LASX 0 +#endif + /** * @ingroup xsimd_config_macro * @@ -626,7 +659,7 @@ #endif -#if !XSIMD_WITH_SSE2 && !XSIMD_WITH_SSE3 && !XSIMD_WITH_SSSE3 && !XSIMD_WITH_SSE4_1 && !XSIMD_WITH_SSE4_2 && !XSIMD_WITH_AVX && !XSIMD_WITH_AVX2 && !XSIMD_WITH_AVXVNNI && !XSIMD_WITH_FMA3_SSE && !XSIMD_WITH_FMA4 && !XSIMD_WITH_FMA3_AVX && !XSIMD_WITH_FMA3_AVX2 && !XSIMD_WITH_AVX512F && !XSIMD_WITH_AVX512CD && !XSIMD_WITH_AVX512VL && !XSIMD_WITH_AVX512DQ && !XSIMD_WITH_AVX512BW && !XSIMD_WITH_AVX512ER && !XSIMD_WITH_AVX512PF && !XSIMD_WITH_AVX512IFMA && !XSIMD_WITH_AVX512VBMI && !XSIMD_WITH_AVX512VBMI2 && !XSIMD_WITH_NEON && !XSIMD_WITH_NEON64 && !XSIMD_WITH_SVE && !XSIMD_WITH_RVV && !XSIMD_WITH_WASM && !XSIMD_WITH_VSX && !XSIMD_WITH_EMULATED && !XSIMD_WITH_VXE +#if !XSIMD_WITH_SSE2 && !XSIMD_WITH_SSE3 && !XSIMD_WITH_SSSE3 && !XSIMD_WITH_SSE4_1 && !XSIMD_WITH_SSE4_2 && !XSIMD_WITH_AVX && !XSIMD_WITH_AVX2 && !XSIMD_WITH_AVXVNNI && !XSIMD_WITH_FMA3_SSE && !XSIMD_WITH_FMA4 && !XSIMD_WITH_FMA3_AVX && !XSIMD_WITH_FMA3_AVX2 && !XSIMD_WITH_AVX512F && !XSIMD_WITH_AVX512CD && !XSIMD_WITH_AVX512VL && !XSIMD_WITH_AVX512DQ && !XSIMD_WITH_AVX512BW && !XSIMD_WITH_AVX512ER && !XSIMD_WITH_AVX512PF && !XSIMD_WITH_AVX512IFMA && !XSIMD_WITH_AVX512VBMI && !XSIMD_WITH_AVX512VBMI2 && !XSIMD_WITH_NEON && !XSIMD_WITH_NEON64 && !XSIMD_WITH_SVE && !XSIMD_WITH_RVV && !XSIMD_WITH_LSX && !XSIMD_WITH_LASX && !XSIMD_WITH_WASM && !XSIMD_WITH_VSX && !XSIMD_WITH_EMULATED && !XSIMD_WITH_VXE #define XSIMD_NO_SUPPORTED_ARCHITECTURE #endif diff --git a/include/xsimd/config/xsimd_cpu_features.hpp b/include/xsimd/config/xsimd_cpu_features.hpp index 5dcc00416..f6035223d 100644 --- a/include/xsimd/config/xsimd_cpu_features.hpp +++ b/include/xsimd/config/xsimd_cpu_features.hpp @@ -15,6 +15,7 @@ #include "./xsimd_cpu_features_arm.hpp" #include "./xsimd_cpu_features_ppc.hpp" #include "./xsimd_cpu_features_riscv.hpp" +#include "./xsimd_cpu_features_loongarch.hpp" #include "./xsimd_cpu_features_s390x.hpp" #include "./xsimd_cpu_features_x86.hpp" @@ -40,6 +41,7 @@ namespace xsimd class cpu_features : public s390x_cpu_features, public ppc_cpu_features, public riscv_cpu_features, + public loongarch_cpu_features, public arm_cpu_features, public x86_cpu_features { diff --git a/include/xsimd/config/xsimd_cpu_features_loongarch.hpp b/include/xsimd/config/xsimd_cpu_features_loongarch.hpp new file mode 100644 index 000000000..fd34e60fd --- /dev/null +++ b/include/xsimd/config/xsimd_cpu_features_loongarch.hpp @@ -0,0 +1,88 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_CPU_FEATURES_LOONGARCH_HPP +#define XSIMD_CPU_FEATURES_LOONGARCH_HPP + +#include "./xsimd_config.hpp" +#include "./xsimd_getauxval.hpp" + +#include +#include + +#if XSIMD_TARGET_LOONGARCH64 && XSIMD_HAVE_LINUX_GETAUXVAL +// HWCAP_XXX masks to use on getauxval results. +// Header does not exists on all architectures and masks are architecture +// specific. +#include +#endif // XSIMD_TARGET_LOONGARCH64 && XSIMD_HAVE_LINUX_GETAUXVAL + +namespace xsimd +{ + /** + * An opinionated CPU feature detection utility for LoongArch. + * + * On Linux, runtime detection uses getauxval to query the auxiliary vector. + * On other platforms, only compile-time information is used. + * + * This is well defined on all architectures. + * It will always return false on non-LoongArch architectures. + */ + class loongarch_cpu_features : private linux_hwcap_backend_default + { + public: + inline bool lsx() const noexcept; + inline bool lasx() const noexcept; + }; + + /******************** + * Implementation * + ********************/ + + inline bool loongarch_cpu_features::lsx() const noexcept + { +#if XSIMD_TARGET_LOONGARCH64 && XSIMD_HAVE_LINUX_GETAUXVAL +#ifdef HWCAP_LOONGARCH_LSX + constexpr unsigned long loongarch_hwcap_lsx = HWCAP_LOONGARCH_LSX; +#else + // Possibly missing on older Linux distributions + constexpr unsigned long loongarch_hwcap_lsx = 1ul << 4; +#endif +#ifdef HWCAP_LOONGARCH_LASX + constexpr unsigned long loongarch_hwcap_lasx = HWCAP_LOONGARCH_LASX; +#else + // Possibly missing on older Linux distributions + constexpr unsigned long loongarch_hwcap_lasx = 1ul << 5; +#endif + return hwcap().has_feature(loongarch_hwcap_lsx) + || hwcap().has_feature(loongarch_hwcap_lasx); +#else + return XSIMD_WITH_LSX || XSIMD_WITH_LASX; +#endif + } + + inline bool loongarch_cpu_features::lasx() const noexcept + { +#if XSIMD_TARGET_LOONGARCH64 && XSIMD_HAVE_LINUX_GETAUXVAL +#ifdef HWCAP_LOONGARCH_LASX + constexpr unsigned long loongarch_hwcap_lasx = HWCAP_LOONGARCH_LASX; +#else + // Possibly missing on older Linux distributions + constexpr unsigned long loongarch_hwcap_lasx = 1ul << 5; +#endif + return hwcap().has_feature(loongarch_hwcap_lasx); +#else + return XSIMD_WITH_LASX; +#endif + } +} + +#endif diff --git a/include/xsimd/config/xsimd_cpuid.hpp b/include/xsimd/config/xsimd_cpuid.hpp index 7466cd5f8..a82816c73 100644 --- a/include/xsimd/config/xsimd_cpuid.hpp +++ b/include/xsimd/config/xsimd_cpuid.hpp @@ -68,6 +68,8 @@ namespace xsimd ARCH_FIELD_EX(detail::rvv<512>, rvv512) ARCH_FIELD_EX(detail::rvv<256>, rvv256) ARCH_FIELD_EX(detail::rvv<128>, rvv128) + ARCH_FIELD(lsx) + ARCH_FIELD(lasx) ARCH_FIELD(wasm) ARCH_FIELD(vsx) ARCH_FIELD(vxe) @@ -90,6 +92,9 @@ namespace xsimd rvv256 = cpu.rvv() && (cpu.rvv_size_bytes() >= (256 / 8)); rvv512 = cpu.rvv() && (cpu.rvv_size_bytes() >= (512 / 8)); + lsx = cpu.lsx(); + lasx = cpu.lasx(); + neon = cpu.neon(); neon64 = cpu.neon64(); i8mm_neon64 = cpu.neon64() && cpu.i8mm(); diff --git a/include/xsimd/types/xsimd_all_registers.hpp b/include/xsimd/types/xsimd_all_registers.hpp index 75d771752..aae59fc35 100644 --- a/include/xsimd/types/xsimd_all_registers.hpp +++ b/include/xsimd/types/xsimd_all_registers.hpp @@ -34,6 +34,8 @@ #include "./xsimd_neon64_register.hpp" #include "./xsimd_neon_register.hpp" #include "./xsimd_rvv_register.hpp" +#include "./xsimd_lasx_register.hpp" +#include "./xsimd_lsx_register.hpp" #include "./xsimd_sse2_register.hpp" #include "./xsimd_sse3_register.hpp" #include "./xsimd_sse4_1_register.hpp" diff --git a/include/xsimd/types/xsimd_lasx_register.hpp b/include/xsimd/types/xsimd_lasx_register.hpp new file mode 100644 index 000000000..cb14331bc --- /dev/null +++ b/include/xsimd/types/xsimd_lasx_register.hpp @@ -0,0 +1,67 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_LASX_REGISTER_HPP +#define XSIMD_LASX_REGISTER_HPP + +#include "./xsimd_lsx_register.hpp" + +#if XSIMD_WITH_LASX && !XSIMD_WITH_LSX +#error "architecture inconsistency: lasx requires lsx" +#endif + +#if XSIMD_WITH_LASX +#include +#endif + +namespace xsimd +{ + /** + * @ingroup architectures + * + * Loongson Advanced SIMD Extension (LASX). + */ + struct lasx : common + { + static constexpr bool supported() noexcept { return XSIMD_WITH_LASX; } + static constexpr bool available() noexcept { return true; } + static constexpr bool requires_alignment() noexcept { return true; } + static constexpr std::size_t alignment() noexcept { return 32; } + static constexpr char const* name() noexcept { return "loongarch64+lasx"; } + }; + +#if XSIMD_WITH_LASX + namespace types + { + XSIMD_DECLARE_SIMD_REGISTER(signed char, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned char, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(char, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(short, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned short, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(int, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned int, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(long, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned long, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(long long, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned long long, lasx, __m256i); + XSIMD_DECLARE_SIMD_REGISTER(float, lasx, __m256); + XSIMD_DECLARE_SIMD_REGISTER(double, lasx, __m256d); + + template + struct get_bool_simd_register + { + using type = simd_register, lasx>; + }; + } +#endif +} + +#endif diff --git a/include/xsimd/types/xsimd_lsx_register.hpp b/include/xsimd/types/xsimd_lsx_register.hpp new file mode 100644 index 000000000..c6abac40c --- /dev/null +++ b/include/xsimd/types/xsimd_lsx_register.hpp @@ -0,0 +1,68 @@ +/*************************************************************************** + * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and * + * Martin Renou * + * Copyright (c) QuantStack * + * Copyright (c) Serge Guelton * + * * + * Distributed under the terms of the BSD 3-Clause License. * + * * + * The full license is in the file LICENSE, distributed with this software. * + ****************************************************************************/ + +#ifndef XSIMD_LSX_REGISTER_HPP +#define XSIMD_LSX_REGISTER_HPP + +#include "../config/xsimd_config.hpp" +#include "../utils/xsimd_type_traits.hpp" +#include "./xsimd_common_arch.hpp" +#include "./xsimd_register.hpp" + +#include + +#if XSIMD_WITH_LSX +#include +#endif + +namespace xsimd +{ + /** + * @ingroup architectures + * + * Loongson SIMD Extension (LSX). + */ + struct lsx : common + { + static constexpr bool supported() noexcept { return XSIMD_WITH_LSX; } + static constexpr bool available() noexcept { return true; } + static constexpr bool requires_alignment() noexcept { return true; } + static constexpr std::size_t alignment() noexcept { return 16; } + static constexpr char const* name() noexcept { return "loongarch64+lsx"; } + }; + +#if XSIMD_WITH_LSX + namespace types + { + XSIMD_DECLARE_SIMD_REGISTER(signed char, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned char, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(char, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(short, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned short, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(int, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned int, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(long, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned long, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(long long, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(unsigned long long, lsx, __m128i); + XSIMD_DECLARE_SIMD_REGISTER(float, lsx, __m128); + XSIMD_DECLARE_SIMD_REGISTER(double, lsx, __m128d); + + template + struct get_bool_simd_register + { + using type = simd_register, lsx>; + }; + } +#endif +} + +#endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a34dafde9..e9826a9b4 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -97,7 +97,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU" message(STATUS "CMAKE_CXX_LINK_EXECUTABLE: ${CMAKE_CXX_LINK_EXECUTABLE}") elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "^ppc64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=${TARGET_ARCH} -mtune=${TARGET_ARCH}") - elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64") + elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64|loongarch64") # Nothing specific elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "powerpc") # Nothing specific diff --git a/test/test_arch.cpp b/test/test_arch.cpp index 3f460bf27..d554930b9 100644 --- a/test/test_arch.cpp +++ b/test/test_arch.cpp @@ -160,7 +160,7 @@ TEST_CASE("[multi arch support]") using batch8i32 = xsimd::make_sized_batch_t; using batch8u32 = xsimd::make_sized_batch_t; -#if XSIMD_WITH_SSE2 || XSIMD_WITH_NEON || XSIMD_WITH_NEON64 || XSIMD_WITH_SVE || (XSIMD_WITH_RVV && XSIMD_RVV_BITS == 128) +#if XSIMD_WITH_SSE2 || XSIMD_WITH_NEON || XSIMD_WITH_NEON64 || XSIMD_WITH_SVE || XSIMD_WITH_LSX || (XSIMD_WITH_RVV && XSIMD_RVV_BITS == 128) CHECK_EQ(4, size_t(batch4f::size)); CHECK_EQ(4, size_t(batch4c::size)); CHECK_EQ(4, size_t(batch4i32::size)); @@ -171,7 +171,7 @@ TEST_CASE("[multi arch support]") CHECK_UNARY(bool(std::is_same_v)); CHECK_UNARY(bool(std::is_same_v)); -#if XSIMD_WITH_SSE2 || XSIMD_WITH_NEON64 || XSIMD_WITH_SVE || XSIMD_WITH_RVV +#if XSIMD_WITH_SSE2 || XSIMD_WITH_NEON64 || XSIMD_WITH_SVE || XSIMD_WITH_RVV || XSIMD_WITH_LSX CHECK_EQ(2, size_t(batch2d::size)); CHECK_EQ(2, size_t(batch2z::size)); CHECK_UNARY(bool(std::is_same_v)); @@ -181,7 +181,7 @@ TEST_CASE("[multi arch support]") #endif #endif -#if !XSIMD_WITH_AVX && !XSIMD_WITH_FMA3 && !(XSIMD_WITH_SVE && XSIMD_SVE_BITS == 256) && !(XSIMD_WITH_RVV && XSIMD_RVV_BITS == 256) +#if !XSIMD_WITH_AVX && !XSIMD_WITH_FMA3 && !XSIMD_WITH_LASX && !(XSIMD_WITH_SVE && XSIMD_SVE_BITS == 256) && !(XSIMD_WITH_RVV && XSIMD_RVV_BITS == 256) CHECK_UNARY(bool(std::is_same_v)); CHECK_UNARY(bool(std::is_same_v)); CHECK_UNARY(bool(std::is_same_v)); diff --git a/test/test_batch_cast.cpp b/test/test_batch_cast.cpp index 954ae0e66..81884948e 100644 --- a/test/test_batch_cast.cpp +++ b/test/test_batch_cast.cpp @@ -396,7 +396,7 @@ TYPED_TEST(batch_cast_test, cast_sizeshift2) } #endif -#if XSIMD_WITH_SSE2 +#if XSIMD_WITH_SSE2 || XSIMD_WITH_LSX || XSIMD_WITH_LASX TEST_CASE_TEMPLATE("[xsimd cast tests]", B, CONVERSION_TYPES) { SUBCASE("use fastcast") diff --git a/test/test_batch_complex.cpp b/test/test_batch_complex.cpp index cccb4bee1..94bb799fd 100644 --- a/test/test_batch_complex.cpp +++ b/test/test_batch_complex.cpp @@ -17,6 +17,7 @@ #include #include #include +#include using namespace std::placeholders; @@ -199,6 +200,11 @@ struct batch_complex_test imag[i] = lhs[i].imag(); tmp[i] = value_type(real[i]); } + +#if XSIMD_WITH_LSX || XSIMD_WITH_LASX || XSIMD_WITH_SVE || XSIMD_WITH_RVV + batch_type b2 = make_batch(lhs, std::make_index_sequence {}); + CHECK_EQ(b2, lhs); +#endif } void test_access_operator() const @@ -704,6 +710,14 @@ struct batch_complex_test #endif private: +#if XSIMD_WITH_LSX || XSIMD_WITH_LASX || XSIMD_WITH_SVE || XSIMD_WITH_RVV + template + batch_type make_batch(array_type const& values, std::index_sequence) const + { + return batch_type(values[I]...); + } +#endif + batch_type batch_lhs() const { batch_type res = batch_type::load_unaligned(lhs.data()); diff --git a/test/test_cpu_features.cpp b/test/test_cpu_features.cpp index 05958a204..cf1c9ad33 100644 --- a/test/test_cpu_features.cpp +++ b/test/test_cpu_features.cpp @@ -172,6 +172,21 @@ TEST_CASE("[cpu_features] risc-v features from environment") CHECK_ENV_FEATURE("XSIMD_TEST_CPU_ASSUME_RVV", cpu.rvv()); } +TEST_CASE("[cpu_features] LoongArch implication chains") +{ + xsimd::cpu_features cpu; + + CHECK_IMPLICATION(cpu.lasx(), cpu.lsx()); +} + +TEST_CASE("[cpu_features] LoongArch features from environment") +{ + xsimd::cpu_features cpu; + + CHECK_ENV_FEATURE("XSIMD_TEST_CPU_ASSUME_LSX", cpu.lsx()); + CHECK_ENV_FEATURE("XSIMD_TEST_CPU_ASSUME_LASX", cpu.lasx()); +} + TEST_CASE("[cpu_features] ppc features from environment") { xsimd::cpu_features cpu;