diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index e85b360c986668c..9e3f788a0bd1a4d 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -4,12 +4,14 @@ // Operations are sequentially consistent unless they have a suffix indicating // otherwise. If in doubt, prefer the sequentially consistent operations. // -// The "_relaxed" suffix for load and store operations indicates the "relaxed" -// memory order. They don't provide synchronization, but (roughly speaking) -// guarantee somewhat sane behavior for races instead of undefined behavior. -// In practice, they correspond to "normal" hardware load and store +// The "_relaxed" suffix indicates the "relaxed" memory order. Relaxed +// operations don't provide synchronization, but (roughly speaking) guarantee +// somewhat sane behavior for races instead of undefined behavior. In practice, +// relaxed loads and stores correspond to "normal" hardware load and store // instructions, so they are almost as inexpensive as plain loads and stores -// in C. +// in C. Relaxed read-modify-write operations, such as +// _Py_atomic_add_*_relaxed, are still atomic, but do not order surrounding +// memory accesses. // // Note that atomic read-modify-write operations like _Py_atomic_add_* return // the previous value of the atomic variable, not the new value. @@ -49,12 +51,27 @@ // expected = obj // return False // +// def _Py_atomic_compare_exchange_relaxed(obj, expected, desired): +// # relaxed consistency +// if obj == expected: +// obj = desired +// return True +// else: +// expected = obj +// return False +// // def _Py_atomic_add(obj, value): // # sequential consistency // old_obj = obj // obj += value // return old_obj // +// def _Py_atomic_add_relaxed(obj, value): +// # relaxed consistency +// old_obj = obj +// obj += value +// return old_obj +// // def _Py_atomic_and(obj, value): // # sequential consistency // old_obj = obj @@ -130,6 +147,50 @@ static inline Py_ssize_t _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value); +// --- _Py_atomic_add_relaxed ------------------------------------------------ +// Atomically adds `value` to `obj` and returns the previous value +// (relaxed consistency, i.e., no ordering) + +static inline int +_Py_atomic_add_int_relaxed(int *obj, int value); + +static inline int8_t +_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value); + +static inline int16_t +_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value); + +static inline int32_t +_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value); + +static inline int64_t +_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value); + +static inline intptr_t +_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value); + +static inline unsigned int +_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value); + +static inline uint8_t +_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value); + +static inline Py_ssize_t +_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value); + + // --- _Py_atomic_compare_exchange ------------------------------------------- // Performs an atomic compare-and-exchange. // @@ -185,6 +246,64 @@ static inline int _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value); +// --- _Py_atomic_compare_exchange_relaxed ----------------------------------- +// Performs an atomic compare-and-exchange (relaxed consistency, i.e., no +// ordering on either success or failure). +// +// - If `*obj` and `*expected` are equal, store `desired` into `*obj` +// and return 1 (success). +// - Otherwise, store the `*obj` current value into `*expected` +// and return 0 (failure). +// +// These correspond to the C11 atomic_compare_exchange_strong_explicit() +// function with memory_order_relaxed for both the success and failure +// memory orders. + +static inline int +_Py_atomic_compare_exchange_int_relaxed(int *obj, int *expected, int desired); + +static inline int +_Py_atomic_compare_exchange_int8_relaxed(int8_t *obj, int8_t *expected, int8_t desired); + +static inline int +_Py_atomic_compare_exchange_int16_relaxed(int16_t *obj, int16_t *expected, int16_t desired); + +static inline int +_Py_atomic_compare_exchange_int32_relaxed(int32_t *obj, int32_t *expected, int32_t desired); + +static inline int +_Py_atomic_compare_exchange_int64_relaxed(int64_t *obj, int64_t *expected, int64_t desired); + +static inline int +_Py_atomic_compare_exchange_intptr_relaxed(intptr_t *obj, intptr_t *expected, intptr_t desired); + +static inline int +_Py_atomic_compare_exchange_uint_relaxed(unsigned int *obj, unsigned int *expected, unsigned int desired); + +static inline int +_Py_atomic_compare_exchange_uint8_relaxed(uint8_t *obj, uint8_t *expected, uint8_t desired); + +static inline int +_Py_atomic_compare_exchange_uint16_relaxed(uint16_t *obj, uint16_t *expected, uint16_t desired); + +static inline int +_Py_atomic_compare_exchange_uint32_relaxed(uint32_t *obj, uint32_t *expected, uint32_t desired); + +static inline int +_Py_atomic_compare_exchange_uint64_relaxed(uint64_t *obj, uint64_t *expected, uint64_t desired); + +static inline int +_Py_atomic_compare_exchange_uintptr_relaxed(uintptr_t *obj, uintptr_t *expected, uintptr_t desired); + +static inline int +_Py_atomic_compare_exchange_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired); + +// NOTE: `obj` and `expected` are logically `void**` types, but we use `void*` +// so that we can pass types like `PyObject**` without a cast. +static inline int +_Py_atomic_compare_exchange_ptr_relaxed(void *obj, void *expected, void *value); + + // --- _Py_atomic_exchange --------------------------------------------------- // Atomically replaces `*obj` with `value` and returns the previous value of `*obj`. diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 253b35082aafcd2..f04f06b51c71870 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -63,6 +63,61 @@ _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) { return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } +// --- _Py_atomic_add_relaxed ------------------------------------------------ + +static inline int +_Py_atomic_add_int_relaxed(int *obj, int value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline int8_t +_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline int16_t +_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline int32_t +_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline int64_t +_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline intptr_t +_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline unsigned int +_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline uint8_t +_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline uint16_t +_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline uint32_t +_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline uint64_t +_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline uintptr_t +_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline Py_ssize_t +_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + + // --- _Py_atomic_compare_exchange ------------------------------------------- static inline int @@ -136,6 +191,79 @@ _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired) __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } +// --- _Py_atomic_compare_exchange_relaxed ----------------------------------- + +static inline int +_Py_atomic_compare_exchange_int_relaxed(int *obj, int *expected, int desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_int8_relaxed(int8_t *obj, int8_t *expected, int8_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_int16_relaxed(int16_t *obj, int16_t *expected, int16_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_int32_relaxed(int32_t *obj, int32_t *expected, int32_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_int64_relaxed(int64_t *obj, int64_t *expected, int64_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_intptr_relaxed(intptr_t *obj, intptr_t *expected, intptr_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_uint_relaxed(unsigned int *obj, unsigned int *expected, unsigned int desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_uint8_relaxed(uint8_t *obj, uint8_t *expected, uint8_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_uint16_relaxed(uint16_t *obj, uint16_t *expected, uint16_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_uint32_relaxed(uint32_t *obj, uint32_t *expected, uint32_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_uint64_relaxed(uint64_t *obj, uint64_t *expected, uint64_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_uintptr_relaxed(uintptr_t *obj, uintptr_t *expected, uintptr_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_ptr_relaxed(void *obj, void *expected, void *desired) +{ return __atomic_compare_exchange_n((void **)obj, (void **)expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + + // --- _Py_atomic_exchange --------------------------------------------------- static inline int diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 3b3c5f7017e9575..c74925d4e2ec0aa 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -125,6 +125,132 @@ _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) } +// --- _Py_atomic_add_relaxed ------------------------------------------------ + +// The "_nf" (no fence) intrinsic variants provide relaxed memory order on +// ARM64. On x86 and x86-64 they do not exist; the plain interlocked +// intrinsics are used instead, which have stronger (sequentially consistent) +// ordering. That is a conforming implementation of relaxed memory order; +// x86 simply has no cheaper atomic read-modify-write. + +static inline int8_t +_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); +#if defined(_M_ARM64) + return (int8_t)_InterlockedExchangeAdd8_nf((volatile char *)obj, (char)value); +#else + return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value); +#endif +} + +static inline int16_t +_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); +#if defined(_M_ARM64) + return (int16_t)_InterlockedExchangeAdd16_nf((volatile short *)obj, (short)value); +#else + return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value); +#endif +} + +static inline int32_t +_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); +#if defined(_M_ARM64) + return (int32_t)_InterlockedExchangeAdd_nf((volatile long *)obj, (long)value); +#else + return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value); +#endif +} + +static inline int64_t +_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value) +{ +#if defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (int64_t)_InterlockedExchangeAdd64_nf((volatile __int64 *)obj, (__int64)value); +#elif defined(_M_X64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value); +#else + int64_t old_value = _Py_atomic_load_int64_relaxed(obj); + for (;;) { + int64_t new_value = old_value + value; + if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + +static inline uint8_t +_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value) +{ + return (uint8_t)_Py_atomic_add_int8_relaxed((int8_t *)obj, (int8_t)value); +} + +static inline uint16_t +_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value) +{ + return (uint16_t)_Py_atomic_add_int16_relaxed((int16_t *)obj, (int16_t)value); +} + +static inline uint32_t +_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value) +{ + return (uint32_t)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value); +} + +static inline int +_Py_atomic_add_int_relaxed(int *obj, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (int)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value); +} + +static inline unsigned int +_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (unsigned int)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value); +} + +static inline uint64_t +_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value) +{ + return (uint64_t)_Py_atomic_add_int64_relaxed((int64_t *)obj, (int64_t)value); +} + +static inline intptr_t +_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(int64_t); + return (intptr_t)_Py_atomic_add_int64_relaxed((int64_t *)obj, (int64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (intptr_t)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value); +#endif +} + +static inline uintptr_t +_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(intptr_t); + return (uintptr_t)_Py_atomic_add_intptr_relaxed((intptr_t *)obj, (intptr_t)value); +} + +static inline Py_ssize_t +_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(intptr_t); + return (Py_ssize_t)_Py_atomic_add_intptr_relaxed((intptr_t *)obj, (intptr_t)value); +} + + // --- _Py_atomic_compare_exchange ------------------------------------------- static inline int @@ -280,6 +406,200 @@ _Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssiz } +// --- _Py_atomic_compare_exchange_relaxed ----------------------------------- + +// As with _Py_atomic_add_*_relaxed, the "_nf" (no fence) intrinsic variants +// provide relaxed memory order on ARM64, and the plain interlocked intrinsics +// stand in for them on x86 and x86-64, where they do not exist. + +static inline int +_Py_atomic_compare_exchange_int8_relaxed(int8_t *obj, int8_t *expected, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); +#if defined(_M_ARM64) + int8_t initial = (int8_t)_InterlockedCompareExchange8_nf( + (volatile char *)obj, + (char)value, + (char)*expected); +#else + int8_t initial = (int8_t)_InterlockedCompareExchange8( + (volatile char *)obj, + (char)value, + (char)*expected); +#endif + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int16_relaxed(int16_t *obj, int16_t *expected, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); +#if defined(_M_ARM64) + int16_t initial = (int16_t)_InterlockedCompareExchange16_nf( + (volatile short *)obj, + (short)value, + (short)*expected); +#else + int16_t initial = (int16_t)_InterlockedCompareExchange16( + (volatile short *)obj, + (short)value, + (short)*expected); +#endif + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int32_relaxed(int32_t *obj, int32_t *expected, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); +#if defined(_M_ARM64) + int32_t initial = (int32_t)_InterlockedCompareExchange_nf( + (volatile long *)obj, + (long)value, + (long)*expected); +#else + int32_t initial = (int32_t)_InterlockedCompareExchange( + (volatile long *)obj, + (long)value, + (long)*expected); +#endif + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int64_relaxed(int64_t *obj, int64_t *expected, int64_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(__int64); +#if defined(_M_ARM64) + int64_t initial = (int64_t)_InterlockedCompareExchange64_nf( + (volatile __int64 *)obj, + (__int64)value, + (__int64)*expected); +#else + int64_t initial = (int64_t)_InterlockedCompareExchange64( + (volatile __int64 *)obj, + (__int64)value, + (__int64)*expected); +#endif + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_ptr_relaxed(void *obj, void *expected, void *value) +{ +#if defined(_M_ARM64) + void *initial = _InterlockedCompareExchangePointer_nf( + (void**)obj, + value, + *(void**)expected); +#else + void *initial = _InterlockedCompareExchangePointer( + (void**)obj, + value, + *(void**)expected); +#endif + if (initial == *(void**)expected) { + return 1; + } + *(void**)expected = initial; + return 0; +} + + +static inline int +_Py_atomic_compare_exchange_uint8_relaxed(uint8_t *obj, uint8_t *expected, uint8_t value) +{ + return _Py_atomic_compare_exchange_int8_relaxed((int8_t *)obj, + (int8_t *)expected, + (int8_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint16_relaxed(uint16_t *obj, uint16_t *expected, uint16_t value) +{ + return _Py_atomic_compare_exchange_int16_relaxed((int16_t *)obj, + (int16_t *)expected, + (int16_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint32_relaxed(uint32_t *obj, uint32_t *expected, uint32_t value) +{ + return _Py_atomic_compare_exchange_int32_relaxed((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_int_relaxed(int *obj, int *expected, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return _Py_atomic_compare_exchange_int32_relaxed((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint_relaxed(unsigned int *obj, unsigned int *expected, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return _Py_atomic_compare_exchange_int32_relaxed((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint64_relaxed(uint64_t *obj, uint64_t *expected, uint64_t value) +{ + return _Py_atomic_compare_exchange_int64_relaxed((int64_t *)obj, + (int64_t *)expected, + (int64_t)value); +} + +static inline int +_Py_atomic_compare_exchange_intptr_relaxed(intptr_t *obj, intptr_t *expected, intptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr_relaxed((void**)obj, + (void**)expected, + (void*)value); +} + +static inline int +_Py_atomic_compare_exchange_uintptr_relaxed(uintptr_t *obj, uintptr_t *expected, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr_relaxed((void**)obj, + (void**)expected, + (void*)value); +} + +static inline int +_Py_atomic_compare_exchange_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr_relaxed((void**)obj, + (void**)expected, + (void*)value); +} + + // --- _Py_atomic_exchange --------------------------------------------------- static inline int8_t diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index faef303da70314c..bb66a8742211b73 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -112,6 +112,113 @@ _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) } +// --- _Py_atomic_add_relaxed ------------------------------------------------ + +static inline int +_Py_atomic_add_int_relaxed(int *obj, int value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(int)*)obj, value, + memory_order_relaxed); +} + +static inline int8_t +_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(int8_t)*)obj, value, + memory_order_relaxed); +} + +static inline int16_t +_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(int16_t)*)obj, value, + memory_order_relaxed); +} + +static inline int32_t +_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(int32_t)*)obj, value, + memory_order_relaxed); +} + +static inline int64_t +_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(int64_t)*)obj, value, + memory_order_relaxed); +} + +static inline intptr_t +_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(intptr_t)*)obj, value, + memory_order_relaxed); +} + +static inline unsigned int +_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_relaxed); +} + +static inline uint8_t +_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(uint8_t)*)obj, value, + memory_order_relaxed); +} + +static inline uint16_t +_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(uint16_t)*)obj, value, + memory_order_relaxed); +} + +static inline uint32_t +_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(uint32_t)*)obj, value, + memory_order_relaxed); +} + +static inline uint64_t +_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(uint64_t)*)obj, value, + memory_order_relaxed); +} + +static inline uintptr_t +_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(uintptr_t)*)obj, value, + memory_order_relaxed); +} + +static inline Py_ssize_t +_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(Py_ssize_t)*)obj, value, + memory_order_relaxed); +} + + // --- _Py_atomic_compare_exchange ------------------------------------------- static inline int @@ -227,6 +334,149 @@ _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired) } +// --- _Py_atomic_compare_exchange_relaxed ----------------------------------- + +static inline int +_Py_atomic_compare_exchange_int_relaxed(int *obj, int *expected, int desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(int)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_int8_relaxed(int8_t *obj, int8_t *expected, int8_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(int8_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_int16_relaxed(int16_t *obj, int16_t *expected, int16_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(int16_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_int32_relaxed(int32_t *obj, int32_t *expected, int32_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(int32_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_int64_relaxed(int64_t *obj, int64_t *expected, int64_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(int64_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_intptr_relaxed(intptr_t *obj, intptr_t *expected, intptr_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(intptr_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_uint_relaxed(unsigned int *obj, unsigned int *expected, unsigned int desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(unsigned int)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_uint8_relaxed(uint8_t *obj, uint8_t *expected, uint8_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(uint8_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_uint16_relaxed(uint16_t *obj, uint16_t *expected, uint16_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(uint16_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_uint32_relaxed(uint32_t *obj, uint32_t *expected, uint32_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(uint32_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_uint64_relaxed(uint64_t *obj, uint64_t *expected, uint64_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(uint64_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_uintptr_relaxed(uintptr_t *obj, uintptr_t *expected, uintptr_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(uintptr_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(Py_ssize_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_ptr_relaxed(void *obj, void *expected, void *desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(void *)*)obj, + (void **)expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + + // --- _Py_atomic_exchange --------------------------------------------------- static inline int diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index d8ec306a0dae3fc..306fb22e1db12e1 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -137,6 +137,8 @@ extern "C" { _Py_atomic_load_ullong_relaxed(&value) #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) +#define FT_ATOMIC_ADD_SSIZE_RELAXED(value, new_value) \ + (void)_Py_atomic_add_ssize_relaxed(&value, new_value) #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) #define FT_MUTEX_LOCK_FLAGS(lock, flags) PyMutex_LockFlags(lock, flags) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) @@ -201,6 +203,7 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) +#define FT_ATOMIC_ADD_SSIZE_RELAXED(value, new_value) (void)(value += new_value) #define FT_MUTEX_LOCK(lock) do {} while (0) #define FT_MUTEX_LOCK_FLAGS(lock, flags) do {} while (0) #define FT_MUTEX_UNLOCK(lock) do {} while (0) diff --git a/Misc/NEWS.d/next/C_API/2026-08-20-15-21-46.gh-issue-155628.9QggKx.rst b/Misc/NEWS.d/next/C_API/2026-08-20-15-21-46.gh-issue-155628.9QggKx.rst new file mode 100644 index 000000000000000..d2008e22320e4aa --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-08-20-15-21-46.gh-issue-155628.9QggKx.rst @@ -0,0 +1,5 @@ +Add relaxed variants of the _Py_atomic_add and _Py_atomic_compare_exchange +functions. The existing operations are sequentially consistent, which is +stronger (and on ARM, measurably more expensive) than necessary for uses like +statistics counters and unique ID allocation, where the operation must be +atomic but does not need to order surrounding memory accesses. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index a380f8ac72b32f4..83faf24f020fe51 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2330,7 +2330,7 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, // store the task counter as PyLong in the name // for deferred formatting in get_name #ifdef Py_GIL_DISABLED - unsigned long long counter = _Py_atomic_add_uint64(&state->task_name_counter, 1) + 1; + unsigned long long counter = _Py_atomic_add_uint64_relaxed(&state->task_name_counter, 1) + 1; #else unsigned long long counter = ++state->task_name_counter; #endif diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 1ab230218124a46..0b3cd687ee037af 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1299,7 +1299,7 @@ uncached_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwd { PyObject *result; - FT_ATOMIC_ADD_SSIZE(self->misses, 1); + FT_ATOMIC_ADD_SSIZE_RELAXED(self->misses, 1); result = PyObject_Call(self->func, args, kwds); if (!result) return NULL; @@ -1321,7 +1321,7 @@ infinite_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwd } int res = _PyDict_GetItemRef_KnownHash((PyDictObject *)self->cache, key, hash, &result); if (res > 0) { - FT_ATOMIC_ADD_SSIZE(self->hits, 1); + FT_ATOMIC_ADD_SSIZE_RELAXED(self->hits, 1); Py_DECREF(key); return result; } @@ -1329,7 +1329,7 @@ infinite_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwd Py_DECREF(key); return NULL; } - FT_ATOMIC_ADD_SSIZE(self->misses, 1); + FT_ATOMIC_ADD_SSIZE_RELAXED(self->misses, 1); result = PyObject_Call(self->func, args, kwds); if (!result) { Py_DECREF(key); @@ -1425,7 +1425,7 @@ bounded_lru_cache_get_lock_held(lru_cache_object *self, PyObject *args, PyObject lru_cache_extract_link(link); lru_cache_append_link(self, link); *result = link->result; - FT_ATOMIC_ADD_SSIZE(self->hits, 1); + FT_ATOMIC_ADD_SSIZE_RELAXED(self->hits, 1); Py_INCREF(link->result); Py_DECREF(link); Py_DECREF(key_); @@ -1435,7 +1435,7 @@ bounded_lru_cache_get_lock_held(lru_cache_object *self, PyObject *args, PyObject Py_DECREF(key_); return -1; } - FT_ATOMIC_ADD_SSIZE(self->misses, 1); + FT_ATOMIC_ADD_SSIZE_RELAXED(self->misses, 1); return 0; } diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index 850de6f9c3366b1..81947b55e9a81d2 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -49,6 +49,12 @@ test_atomic_add_##suffix(PyObject *self, PyObject *obj) { \ assert(x == (dtype)-3); \ assert(_Py_atomic_add_##suffix(&x, 2) == (dtype)-3); \ assert(x == (dtype)-1); \ + assert(_Py_atomic_add_##suffix##_relaxed(&x, 1) == (dtype)-1); \ + assert(x == 0); \ + assert(_Py_atomic_add_##suffix##_relaxed(&x, 3) == 0); \ + assert(x == 3); \ + assert(_Py_atomic_add_##suffix##_relaxed(&x, -4) == 3); \ + assert(x == (dtype)-1); \ Py_RETURN_NONE; \ } FOR_ARITHMETIC_TYPES(IMPL_TEST_ADD) @@ -68,6 +74,17 @@ test_atomic_compare_exchange_##suffix(PyObject *self, PyObject *obj) { \ assert(_Py_atomic_compare_exchange_##suffix(&x, &y, z) == 0); \ assert(x == z); \ assert(y == z); \ + x = (dtype)0; \ + y = (dtype)1; \ + assert(_Py_atomic_compare_exchange_##suffix##_relaxed(&x, &y, z) == 0); \ + assert(x == 0); \ + assert(y == 0); \ + assert(_Py_atomic_compare_exchange_##suffix##_relaxed(&x, &y, z) == 1); \ + assert(x == z); \ + assert(y == 0); \ + assert(_Py_atomic_compare_exchange_##suffix##_relaxed(&x, &y, z) == 0); \ + assert(x == z); \ + assert(y == z); \ Py_RETURN_NONE; \ } FOR_ALL_TYPES(IMPL_TEST_COMPARE_EXCHANGE) diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index e2df31556f3c372..1042cdf3300c112 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -222,7 +222,7 @@ gc_get_count_impl(PyObject *module) struct _gc_thread_state *gc = &tstate->gc; // Flush the local allocation count to the global count - _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); + _Py_atomic_add_int_relaxed(&gcstate->young.count, (int)gc->alloc_count); gc->alloc_count = 0; #endif diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index f865ac2f7db0853..164bb4baebab167 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2029,7 +2029,7 @@ record_allocation(PyThreadState *tstate) if (gc->alloc_count >= LOCAL_ALLOC_COUNT_THRESHOLD) { // TODO: Use Py_ssize_t for the generation count. GCState *gcstate = &tstate->interp->gc; - _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); + _Py_atomic_add_int_relaxed(&gcstate->young.count, (int)gc->alloc_count); gc->alloc_count = 0; if (gc_should_collect(gcstate) && @@ -2058,9 +2058,9 @@ record_deallocation(PyThreadState *tstate) if (new_count < 0) { new_count = 0; } - } while (!_Py_atomic_compare_exchange_int(&gcstate->young.count, - &count, - new_count)); + } while (!_Py_atomic_compare_exchange_int_relaxed(&gcstate->young.count, + &count, + new_count)); gc->alloc_count = 0; } } diff --git a/Python/import.c b/Python/import.c index 037f15d4ca2bafa..6fcbaaa2743945d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -523,7 +523,7 @@ remove_module(PyThreadState *tstate, PyObject *name) Py_ssize_t _PyImport_GetNextModuleIndex(void) { - return _Py_atomic_add_ssize(&LAST_MODULE_INDEX, 1) + 1; + return _Py_atomic_add_ssize_relaxed(&LAST_MODULE_INDEX, 1) + 1; } #ifndef NDEBUG diff --git a/Python/pystate.c b/Python/pystate.c index 9a2dc9431f8bb24..769205b813fee2b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1881,8 +1881,8 @@ PyThreadState_Clear(PyThreadState *tstate) // Flush the thread's local GC allocation count to the global count // before the thread state is cleared, otherwise the count is lost. _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; - _Py_atomic_add_int(&tstate->interp->gc.young.count, - (int)tstate_impl->gc.alloc_count); + _Py_atomic_add_int_relaxed(&tstate->interp->gc.young.count, + (int)tstate_impl->gc.alloc_count); tstate_impl->gc.alloc_count = 0; // Merge our thread-local refcounts into the type's own refcount and