From e82cf63390ae2100d6a2b01ef675862e1dcadab0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 12 Aug 2026 10:16:32 -0400 Subject: [PATCH 1/7] gh-155628: Add relaxed _Py_atomic_add_* Add _Py_atomic_add_*_relaxed() for all arithmetic types supported by _Py_atomic_add_*(). The existing add operations are sequentially consistent, which is stronger (and on ARM, measurably more expensive) than necessary for uses like statistics counters and unique ID allocation, where the add must be atomic but does not need to order surrounding memory accesses. The GCC/Clang backend uses __atomic_fetch_add() with __ATOMIC_RELAXED, and the standard C11/C++11 backend uses atomic_fetch_add_explicit() with memory_order_relaxed. The MSVC backend uses the _InterlockedExchangeAdd*_nf ("no fence") intrinsics on ARM64; on x86 and x64 those intrinsics do not exist, so it falls back to the plain interlocked intrinsics, whose stronger ordering is a conforming implementation of relaxed (x86 has no cheaper atomic read-modify-write). As with the sequentially consistent version, 64-bit adds on 32-bit x86 fall back to a compare-exchange loop. The _testcapi smoke tests for atomic adds now exercise the relaxed variants as well. --- Include/cpython/pyatomic.h | 62 ++++++++++++++-- Include/cpython/pyatomic_gcc.h | 55 ++++++++++++++ Include/cpython/pyatomic_msc.h | 126 +++++++++++++++++++++++++++++++++ Include/cpython/pyatomic_std.h | 107 ++++++++++++++++++++++++++++ Modules/_testcapi/pyatomic.c | 6 ++ 5 files changed, 351 insertions(+), 5 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index e85b360c986668c..9aa3707b0c07b2e 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -4,12 +4,14 @@ // Operations are sequentially consistent unless they have a suffix indicating // otherwise. If in doubt, prefer the sequentially consistent operations. // -// The "_relaxed" suffix for load and store operations indicates the "relaxed" -// memory order. They don't provide synchronization, but (roughly speaking) -// guarantee somewhat sane behavior for races instead of undefined behavior. -// In practice, they correspond to "normal" hardware load and store +// The "_relaxed" suffix indicates the "relaxed" memory order. Relaxed +// operations don't provide synchronization, but (roughly speaking) guarantee +// somewhat sane behavior for races instead of undefined behavior. In practice, +// relaxed loads and stores correspond to "normal" hardware load and store // instructions, so they are almost as inexpensive as plain loads and stores -// in C. +// in C. Relaxed read-modify-write operations, such as +// _Py_atomic_add_*_relaxed, are still atomic, but do not order surrounding +// memory accesses. // // Note that atomic read-modify-write operations like _Py_atomic_add_* return // the previous value of the atomic variable, not the new value. @@ -55,6 +57,12 @@ // obj += value // return old_obj // +// def _Py_atomic_add_relaxed(obj, value): +// # relaxed consistency +// old_obj = obj +// obj += value +// return old_obj +// // def _Py_atomic_and(obj, value): // # sequential consistency // old_obj = obj @@ -130,6 +138,50 @@ static inline Py_ssize_t _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value); +// --- _Py_atomic_add_relaxed ------------------------------------------------ +// Atomically adds `value` to `obj` and returns the previous value +// (relaxed consistency, i.e., no ordering) + +static inline int +_Py_atomic_add_int_relaxed(int *obj, int value); + +static inline int8_t +_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value); + +static inline int16_t +_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value); + +static inline int32_t +_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value); + +static inline int64_t +_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value); + +static inline intptr_t +_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value); + +static inline unsigned int +_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value); + +static inline uint8_t +_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value); + +static inline uint16_t +_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value); + +static inline uint32_t +_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value); + +static inline uint64_t +_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value); + +static inline uintptr_t +_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value); + +static inline Py_ssize_t +_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value); + + // --- _Py_atomic_compare_exchange ------------------------------------------- // Performs an atomic compare-and-exchange. // diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 253b35082aafcd2..3b14bf6bd82bdbc 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -63,6 +63,61 @@ _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) { return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } +// --- _Py_atomic_add_relaxed ------------------------------------------------ + +static inline int +_Py_atomic_add_int_relaxed(int *obj, int value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline int8_t +_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline int16_t +_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline int32_t +_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline int64_t +_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline intptr_t +_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline unsigned int +_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline uint8_t +_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline uint16_t +_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline uint32_t +_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline uint64_t +_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline uintptr_t +_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + +static inline Py_ssize_t +_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_RELAXED); } + + // --- _Py_atomic_compare_exchange ------------------------------------------- static inline int diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 3b3c5f7017e9575..25991715901b4fc 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -125,6 +125,132 @@ _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) } +// --- _Py_atomic_add_relaxed ------------------------------------------------ + +// The "_nf" (no fence) intrinsic variants provide relaxed memory order on +// ARM64. On x86 and x86-64 they do not exist; the plain interlocked +// intrinsics are used instead, which have stronger (sequentially consistent) +// ordering. That is a conforming implementation of relaxed memory order; +// x86 simply has no cheaper atomic read-modify-write. + +static inline int8_t +_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); +#if defined(_M_ARM64) + return (int8_t)_InterlockedExchangeAdd8_nf((volatile char *)obj, (char)value); +#else + return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value); +#endif +} + +static inline int16_t +_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); +#if defined(_M_ARM64) + return (int16_t)_InterlockedExchangeAdd16_nf((volatile short *)obj, (short)value); +#else + return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value); +#endif +} + +static inline int32_t +_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); +#if defined(_M_ARM64) + return (int32_t)_InterlockedExchangeAdd_nf((volatile long *)obj, (long)value); +#else + return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value); +#endif +} + +static inline int64_t +_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value) +{ +#if defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (int64_t)_InterlockedExchangeAdd64_nf((volatile __int64 *)obj, (__int64)value); +#elif defined(_M_X64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value); +#else + int64_t old_value = _Py_atomic_load_int64_relaxed(obj); + for (;;) { + int64_t new_value = old_value + value; + if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + +static inline uint8_t +_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value) +{ + return (uint8_t)_Py_atomic_add_int8_relaxed((int8_t *)obj, (int8_t)value); +} + +static inline uint16_t +_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value) +{ + return (uint16_t)_Py_atomic_add_int16_relaxed((int16_t *)obj, (int16_t)value); +} + +static inline uint32_t +_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value) +{ + return (uint32_t)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value); +} + +static inline int +_Py_atomic_add_int_relaxed(int *obj, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (int)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value); +} + +static inline unsigned int +_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (unsigned int)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value); +} + +static inline uint64_t +_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value) +{ + return (uint64_t)_Py_atomic_add_int64_relaxed((int64_t *)obj, (int64_t)value); +} + +static inline intptr_t +_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(int64_t); + return (intptr_t)_Py_atomic_add_int64_relaxed((int64_t *)obj, (int64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (intptr_t)_Py_atomic_add_int32_relaxed((int32_t *)obj, (int32_t)value); +#endif +} + +static inline uintptr_t +_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(intptr_t); + return (uintptr_t)_Py_atomic_add_intptr_relaxed((intptr_t *)obj, (intptr_t)value); +} + +static inline Py_ssize_t +_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(intptr_t); + return (Py_ssize_t)_Py_atomic_add_intptr_relaxed((intptr_t *)obj, (intptr_t)value); +} + + // --- _Py_atomic_compare_exchange ------------------------------------------- static inline int diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index faef303da70314c..5298f00af07b049 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -112,6 +112,113 @@ _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) } +// --- _Py_atomic_add_relaxed ------------------------------------------------ + +static inline int +_Py_atomic_add_int_relaxed(int *obj, int value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(int)*)obj, value, + memory_order_relaxed); +} + +static inline int8_t +_Py_atomic_add_int8_relaxed(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(int8_t)*)obj, value, + memory_order_relaxed); +} + +static inline int16_t +_Py_atomic_add_int16_relaxed(int16_t *obj, int16_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(int16_t)*)obj, value, + memory_order_relaxed); +} + +static inline int32_t +_Py_atomic_add_int32_relaxed(int32_t *obj, int32_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(int32_t)*)obj, value, + memory_order_relaxed); +} + +static inline int64_t +_Py_atomic_add_int64_relaxed(int64_t *obj, int64_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(int64_t)*)obj, value, + memory_order_relaxed); +} + +static inline intptr_t +_Py_atomic_add_intptr_relaxed(intptr_t *obj, intptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(intptr_t)*)obj, value, + memory_order_relaxed); +} + +static inline unsigned int +_Py_atomic_add_uint_relaxed(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_relaxed); +} + +static inline uint8_t +_Py_atomic_add_uint8_relaxed(uint8_t *obj, uint8_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(uint8_t)*)obj, value, + memory_order_relaxed); +} + +static inline uint16_t +_Py_atomic_add_uint16_relaxed(uint16_t *obj, uint16_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(uint16_t)*)obj, value, + memory_order_relaxed); +} + +static inline uint32_t +_Py_atomic_add_uint32_relaxed(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(uint32_t)*)obj, value, + memory_order_relaxed); +} + +static inline uint64_t +_Py_atomic_add_uint64_relaxed(uint64_t *obj, uint64_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(uint64_t)*)obj, value, + memory_order_relaxed); +} + +static inline uintptr_t +_Py_atomic_add_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(uintptr_t)*)obj, value, + memory_order_relaxed); +} + +static inline Py_ssize_t +_Py_atomic_add_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_USING_STD; + return atomic_fetch_add_explicit((_Atomic(Py_ssize_t)*)obj, value, + memory_order_relaxed); +} + + // --- _Py_atomic_compare_exchange ------------------------------------------- static inline int diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index 850de6f9c3366b1..4f884af93592055 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -49,6 +49,12 @@ test_atomic_add_##suffix(PyObject *self, PyObject *obj) { \ assert(x == (dtype)-3); \ assert(_Py_atomic_add_##suffix(&x, 2) == (dtype)-3); \ assert(x == (dtype)-1); \ + assert(_Py_atomic_add_##suffix##_relaxed(&x, 1) == (dtype)-1); \ + assert(x == 0); \ + assert(_Py_atomic_add_##suffix##_relaxed(&x, 3) == 0); \ + assert(x == 3); \ + assert(_Py_atomic_add_##suffix##_relaxed(&x, -4) == 3); \ + assert(x == (dtype)-1); \ Py_RETURN_NONE; \ } FOR_ARITHMETIC_TYPES(IMPL_TEST_ADD) From 5fb52a60d1eaf2a2934c181677acb26516f91535 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 12 Aug 2026 10:42:22 -0400 Subject: [PATCH 2/7] gh-155628: Use a relaxed add for the module index allocator The add is the only access to LAST_MODULE_INDEX anywhere in the codebase; only the uniqueness of each returned index matters, which atomicity alone guarantees. --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/import.c b/Python/import.c index 037f15d4ca2bafa..6fcbaaa2743945d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -523,7 +523,7 @@ remove_module(PyThreadState *tstate, PyObject *name) Py_ssize_t _PyImport_GetNextModuleIndex(void) { - return _Py_atomic_add_ssize(&LAST_MODULE_INDEX, 1) + 1; + return _Py_atomic_add_ssize_relaxed(&LAST_MODULE_INDEX, 1) + 1; } #ifndef NDEBUG From fbbfd6cbd16ecfa0fa40367df462627b030272b7 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 12 Aug 2026 10:49:11 -0400 Subject: [PATCH 3/7] gh-155628: Use relaxed adds for the GC allocation counter The free-threaded build buffers per-thread allocation counts and flushes them to gcstate->young.count in three places: when the local threshold is reached, when a thread state is cleared, and in gc.get_count(). The counter is a collection heuristic: its readers use relaxed loads (gc_should_collect()) or a compare-exchange loop, it is reset during a stop-the-world pause, and it publishes no other memory, so the flushes need atomicity but no ordering. --- Modules/gcmodule.c | 2 +- Python/gc_free_threading.c | 2 +- Python/pystate.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index e2df31556f3c372..1042cdf3300c112 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -222,7 +222,7 @@ gc_get_count_impl(PyObject *module) struct _gc_thread_state *gc = &tstate->gc; // Flush the local allocation count to the global count - _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); + _Py_atomic_add_int_relaxed(&gcstate->young.count, (int)gc->alloc_count); gc->alloc_count = 0; #endif diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index f865ac2f7db0853..d88bc538d186594 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2029,7 +2029,7 @@ record_allocation(PyThreadState *tstate) if (gc->alloc_count >= LOCAL_ALLOC_COUNT_THRESHOLD) { // TODO: Use Py_ssize_t for the generation count. GCState *gcstate = &tstate->interp->gc; - _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); + _Py_atomic_add_int_relaxed(&gcstate->young.count, (int)gc->alloc_count); gc->alloc_count = 0; if (gc_should_collect(gcstate) && diff --git a/Python/pystate.c b/Python/pystate.c index 9a2dc9431f8bb24..769205b813fee2b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1881,8 +1881,8 @@ PyThreadState_Clear(PyThreadState *tstate) // Flush the thread's local GC allocation count to the global count // before the thread state is cleared, otherwise the count is lost. _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; - _Py_atomic_add_int(&tstate->interp->gc.young.count, - (int)tstate_impl->gc.alloc_count); + _Py_atomic_add_int_relaxed(&tstate->interp->gc.young.count, + (int)tstate_impl->gc.alloc_count); tstate_impl->gc.alloc_count = 0; // Merge our thread-local refcounts into the type's own refcount and From 38a37ef950a0300349fdcd74eb1face987143692 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 12 Aug 2026 10:51:57 -0400 Subject: [PATCH 4/7] gh-155628: Use relaxed adds for the lru_cache hit/miss counters Add an FT_ATOMIC_ADD_SSIZE_RELAXED wrapper and use it for the lru_cache hits and misses counters, which are updated on every cached call in the free-threaded build. The counters are pure statistics: their only readers are cache_info() and cache_clear(), which already use relaxed loads, so the adds need atomicity but no ordering. --- Include/internal/pycore_pyatomic_ft_wrappers.h | 3 +++ Modules/_functoolsmodule.c | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index d8ec306a0dae3fc..306fb22e1db12e1 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -137,6 +137,8 @@ extern "C" { _Py_atomic_load_ullong_relaxed(&value) #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) +#define FT_ATOMIC_ADD_SSIZE_RELAXED(value, new_value) \ + (void)_Py_atomic_add_ssize_relaxed(&value, new_value) #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) #define FT_MUTEX_LOCK_FLAGS(lock, flags) PyMutex_LockFlags(lock, flags) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) @@ -201,6 +203,7 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) +#define FT_ATOMIC_ADD_SSIZE_RELAXED(value, new_value) (void)(value += new_value) #define FT_MUTEX_LOCK(lock) do {} while (0) #define FT_MUTEX_LOCK_FLAGS(lock, flags) do {} while (0) #define FT_MUTEX_UNLOCK(lock) do {} while (0) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 1ab230218124a46..0b3cd687ee037af 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1299,7 +1299,7 @@ uncached_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwd { PyObject *result; - FT_ATOMIC_ADD_SSIZE(self->misses, 1); + FT_ATOMIC_ADD_SSIZE_RELAXED(self->misses, 1); result = PyObject_Call(self->func, args, kwds); if (!result) return NULL; @@ -1321,7 +1321,7 @@ infinite_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwd } int res = _PyDict_GetItemRef_KnownHash((PyDictObject *)self->cache, key, hash, &result); if (res > 0) { - FT_ATOMIC_ADD_SSIZE(self->hits, 1); + FT_ATOMIC_ADD_SSIZE_RELAXED(self->hits, 1); Py_DECREF(key); return result; } @@ -1329,7 +1329,7 @@ infinite_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwd Py_DECREF(key); return NULL; } - FT_ATOMIC_ADD_SSIZE(self->misses, 1); + FT_ATOMIC_ADD_SSIZE_RELAXED(self->misses, 1); result = PyObject_Call(self->func, args, kwds); if (!result) { Py_DECREF(key); @@ -1425,7 +1425,7 @@ bounded_lru_cache_get_lock_held(lru_cache_object *self, PyObject *args, PyObject lru_cache_extract_link(link); lru_cache_append_link(self, link); *result = link->result; - FT_ATOMIC_ADD_SSIZE(self->hits, 1); + FT_ATOMIC_ADD_SSIZE_RELAXED(self->hits, 1); Py_INCREF(link->result); Py_DECREF(link); Py_DECREF(key_); @@ -1435,7 +1435,7 @@ bounded_lru_cache_get_lock_held(lru_cache_object *self, PyObject *args, PyObject Py_DECREF(key_); return -1; } - FT_ATOMIC_ADD_SSIZE(self->misses, 1); + FT_ATOMIC_ADD_SSIZE_RELAXED(self->misses, 1); return 0; } From cf2b12c5dde37e20088a6e752586db62cc4645ca Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 12 Aug 2026 10:53:28 -0400 Subject: [PATCH 5/7] gh-155628: Use a relaxed add for the asyncio task name counter The counter only generates unique default Task names; the add is its sole access in the free-threaded build, so only the uniqueness of each returned value matters, which atomicity alone guarantees. --- .../C_API/2026-08-20-15-21-46.gh-issue-155628.9QggKx.rst | 5 +++++ Modules/_asynciomodule.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-08-20-15-21-46.gh-issue-155628.9QggKx.rst diff --git a/Misc/NEWS.d/next/C_API/2026-08-20-15-21-46.gh-issue-155628.9QggKx.rst b/Misc/NEWS.d/next/C_API/2026-08-20-15-21-46.gh-issue-155628.9QggKx.rst new file mode 100644 index 000000000000000..b904720a376790c --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-08-20-15-21-46.gh-issue-155628.9QggKx.rst @@ -0,0 +1,5 @@ +Add relaxed variants of the _Py_atomic_add functions. The existing add +operations are sequentially consistent, which is stronger (and on ARM, +measurably more expensive) than necessary for uses like statistics counters +and unique ID allocation, where the add must be atomic but does not need to +order surrounding memory accesses. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index a380f8ac72b32f4..83faf24f020fe51 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2330,7 +2330,7 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, // store the task counter as PyLong in the name // for deferred formatting in get_name #ifdef Py_GIL_DISABLED - unsigned long long counter = _Py_atomic_add_uint64(&state->task_name_counter, 1) + 1; + unsigned long long counter = _Py_atomic_add_uint64_relaxed(&state->task_name_counter, 1) + 1; #else unsigned long long counter = ++state->task_name_counter; #endif From b5f78430868a13f37bfde1c66828d57b53ad3c49 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 9 Sep 2026 12:41:57 -0400 Subject: [PATCH 6/7] gh-155628: Add relaxed _Py_atomic_compare_exchange Add _Py_atomic_compare_exchange_*_relaxed() for all arithmetic types supported by _Py_atomic_compare_exchange_*(). --- Include/cpython/pyatomic.h | 67 ++++++ Include/cpython/pyatomic_gcc.h | 73 +++++++ Include/cpython/pyatomic_msc.h | 194 ++++++++++++++++++ Include/cpython/pyatomic_std.h | 143 +++++++++++++ ...-08-20-15-21-46.gh-issue-155628.9QggKx.rst | 10 +- Modules/_testcapi/pyatomic.c | 11 + 6 files changed, 493 insertions(+), 5 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 9aa3707b0c07b2e..9e3f788a0bd1a4d 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -51,6 +51,15 @@ // expected = obj // return False // +// def _Py_atomic_compare_exchange_relaxed(obj, expected, desired): +// # relaxed consistency +// if obj == expected: +// obj = desired +// return True +// else: +// expected = obj +// return False +// // def _Py_atomic_add(obj, value): // # sequential consistency // old_obj = obj @@ -237,6 +246,64 @@ static inline int _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value); +// --- _Py_atomic_compare_exchange_relaxed ----------------------------------- +// Performs an atomic compare-and-exchange (relaxed consistency, i.e., no +// ordering on either success or failure). +// +// - If `*obj` and `*expected` are equal, store `desired` into `*obj` +// and return 1 (success). +// - Otherwise, store the `*obj` current value into `*expected` +// and return 0 (failure). +// +// These correspond to the C11 atomic_compare_exchange_strong_explicit() +// function with memory_order_relaxed for both the success and failure +// memory orders. + +static inline int +_Py_atomic_compare_exchange_int_relaxed(int *obj, int *expected, int desired); + +static inline int +_Py_atomic_compare_exchange_int8_relaxed(int8_t *obj, int8_t *expected, int8_t desired); + +static inline int +_Py_atomic_compare_exchange_int16_relaxed(int16_t *obj, int16_t *expected, int16_t desired); + +static inline int +_Py_atomic_compare_exchange_int32_relaxed(int32_t *obj, int32_t *expected, int32_t desired); + +static inline int +_Py_atomic_compare_exchange_int64_relaxed(int64_t *obj, int64_t *expected, int64_t desired); + +static inline int +_Py_atomic_compare_exchange_intptr_relaxed(intptr_t *obj, intptr_t *expected, intptr_t desired); + +static inline int +_Py_atomic_compare_exchange_uint_relaxed(unsigned int *obj, unsigned int *expected, unsigned int desired); + +static inline int +_Py_atomic_compare_exchange_uint8_relaxed(uint8_t *obj, uint8_t *expected, uint8_t desired); + +static inline int +_Py_atomic_compare_exchange_uint16_relaxed(uint16_t *obj, uint16_t *expected, uint16_t desired); + +static inline int +_Py_atomic_compare_exchange_uint32_relaxed(uint32_t *obj, uint32_t *expected, uint32_t desired); + +static inline int +_Py_atomic_compare_exchange_uint64_relaxed(uint64_t *obj, uint64_t *expected, uint64_t desired); + +static inline int +_Py_atomic_compare_exchange_uintptr_relaxed(uintptr_t *obj, uintptr_t *expected, uintptr_t desired); + +static inline int +_Py_atomic_compare_exchange_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired); + +// NOTE: `obj` and `expected` are logically `void**` types, but we use `void*` +// so that we can pass types like `PyObject**` without a cast. +static inline int +_Py_atomic_compare_exchange_ptr_relaxed(void *obj, void *expected, void *value); + + // --- _Py_atomic_exchange --------------------------------------------------- // Atomically replaces `*obj` with `value` and returns the previous value of `*obj`. diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 3b14bf6bd82bdbc..f04f06b51c71870 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -191,6 +191,79 @@ _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired) __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } +// --- _Py_atomic_compare_exchange_relaxed ----------------------------------- + +static inline int +_Py_atomic_compare_exchange_int_relaxed(int *obj, int *expected, int desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_int8_relaxed(int8_t *obj, int8_t *expected, int8_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_int16_relaxed(int16_t *obj, int16_t *expected, int16_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_int32_relaxed(int32_t *obj, int32_t *expected, int32_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_int64_relaxed(int64_t *obj, int64_t *expected, int64_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_intptr_relaxed(intptr_t *obj, intptr_t *expected, intptr_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_uint_relaxed(unsigned int *obj, unsigned int *expected, unsigned int desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_uint8_relaxed(uint8_t *obj, uint8_t *expected, uint8_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_uint16_relaxed(uint16_t *obj, uint16_t *expected, uint16_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_uint32_relaxed(uint32_t *obj, uint32_t *expected, uint32_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_uint64_relaxed(uint64_t *obj, uint64_t *expected, uint64_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_uintptr_relaxed(uintptr_t *obj, uintptr_t *expected, uintptr_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired) +{ return __atomic_compare_exchange_n(obj, expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + +static inline int +_Py_atomic_compare_exchange_ptr_relaxed(void *obj, void *expected, void *desired) +{ return __atomic_compare_exchange_n((void **)obj, (void **)expected, desired, 0, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); } + + // --- _Py_atomic_exchange --------------------------------------------------- static inline int diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 25991715901b4fc..c74925d4e2ec0aa 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -406,6 +406,200 @@ _Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssiz } +// --- _Py_atomic_compare_exchange_relaxed ----------------------------------- + +// As with _Py_atomic_add_*_relaxed, the "_nf" (no fence) intrinsic variants +// provide relaxed memory order on ARM64, and the plain interlocked intrinsics +// stand in for them on x86 and x86-64, where they do not exist. + +static inline int +_Py_atomic_compare_exchange_int8_relaxed(int8_t *obj, int8_t *expected, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); +#if defined(_M_ARM64) + int8_t initial = (int8_t)_InterlockedCompareExchange8_nf( + (volatile char *)obj, + (char)value, + (char)*expected); +#else + int8_t initial = (int8_t)_InterlockedCompareExchange8( + (volatile char *)obj, + (char)value, + (char)*expected); +#endif + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int16_relaxed(int16_t *obj, int16_t *expected, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); +#if defined(_M_ARM64) + int16_t initial = (int16_t)_InterlockedCompareExchange16_nf( + (volatile short *)obj, + (short)value, + (short)*expected); +#else + int16_t initial = (int16_t)_InterlockedCompareExchange16( + (volatile short *)obj, + (short)value, + (short)*expected); +#endif + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int32_relaxed(int32_t *obj, int32_t *expected, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); +#if defined(_M_ARM64) + int32_t initial = (int32_t)_InterlockedCompareExchange_nf( + (volatile long *)obj, + (long)value, + (long)*expected); +#else + int32_t initial = (int32_t)_InterlockedCompareExchange( + (volatile long *)obj, + (long)value, + (long)*expected); +#endif + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int64_relaxed(int64_t *obj, int64_t *expected, int64_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(__int64); +#if defined(_M_ARM64) + int64_t initial = (int64_t)_InterlockedCompareExchange64_nf( + (volatile __int64 *)obj, + (__int64)value, + (__int64)*expected); +#else + int64_t initial = (int64_t)_InterlockedCompareExchange64( + (volatile __int64 *)obj, + (__int64)value, + (__int64)*expected); +#endif + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_ptr_relaxed(void *obj, void *expected, void *value) +{ +#if defined(_M_ARM64) + void *initial = _InterlockedCompareExchangePointer_nf( + (void**)obj, + value, + *(void**)expected); +#else + void *initial = _InterlockedCompareExchangePointer( + (void**)obj, + value, + *(void**)expected); +#endif + if (initial == *(void**)expected) { + return 1; + } + *(void**)expected = initial; + return 0; +} + + +static inline int +_Py_atomic_compare_exchange_uint8_relaxed(uint8_t *obj, uint8_t *expected, uint8_t value) +{ + return _Py_atomic_compare_exchange_int8_relaxed((int8_t *)obj, + (int8_t *)expected, + (int8_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint16_relaxed(uint16_t *obj, uint16_t *expected, uint16_t value) +{ + return _Py_atomic_compare_exchange_int16_relaxed((int16_t *)obj, + (int16_t *)expected, + (int16_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint32_relaxed(uint32_t *obj, uint32_t *expected, uint32_t value) +{ + return _Py_atomic_compare_exchange_int32_relaxed((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_int_relaxed(int *obj, int *expected, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return _Py_atomic_compare_exchange_int32_relaxed((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint_relaxed(unsigned int *obj, unsigned int *expected, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return _Py_atomic_compare_exchange_int32_relaxed((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint64_relaxed(uint64_t *obj, uint64_t *expected, uint64_t value) +{ + return _Py_atomic_compare_exchange_int64_relaxed((int64_t *)obj, + (int64_t *)expected, + (int64_t)value); +} + +static inline int +_Py_atomic_compare_exchange_intptr_relaxed(intptr_t *obj, intptr_t *expected, intptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr_relaxed((void**)obj, + (void**)expected, + (void*)value); +} + +static inline int +_Py_atomic_compare_exchange_uintptr_relaxed(uintptr_t *obj, uintptr_t *expected, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr_relaxed((void**)obj, + (void**)expected, + (void*)value); +} + +static inline int +_Py_atomic_compare_exchange_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr_relaxed((void**)obj, + (void**)expected, + (void*)value); +} + + // --- _Py_atomic_exchange --------------------------------------------------- static inline int8_t diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 5298f00af07b049..bb66a8742211b73 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -334,6 +334,149 @@ _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired) } +// --- _Py_atomic_compare_exchange_relaxed ----------------------------------- + +static inline int +_Py_atomic_compare_exchange_int_relaxed(int *obj, int *expected, int desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(int)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_int8_relaxed(int8_t *obj, int8_t *expected, int8_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(int8_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_int16_relaxed(int16_t *obj, int16_t *expected, int16_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(int16_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_int32_relaxed(int32_t *obj, int32_t *expected, int32_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(int32_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_int64_relaxed(int64_t *obj, int64_t *expected, int64_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(int64_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_intptr_relaxed(intptr_t *obj, intptr_t *expected, intptr_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(intptr_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_uint_relaxed(unsigned int *obj, unsigned int *expected, unsigned int desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(unsigned int)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_uint8_relaxed(uint8_t *obj, uint8_t *expected, uint8_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(uint8_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_uint16_relaxed(uint16_t *obj, uint16_t *expected, uint16_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(uint16_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_uint32_relaxed(uint32_t *obj, uint32_t *expected, uint32_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(uint32_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_uint64_relaxed(uint64_t *obj, uint64_t *expected, uint64_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(uint64_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_uintptr_relaxed(uintptr_t *obj, uintptr_t *expected, uintptr_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(uintptr_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(Py_ssize_t)*)obj, + expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + +static inline int +_Py_atomic_compare_exchange_ptr_relaxed(void *obj, void *expected, void *desired) +{ + _Py_USING_STD; + return atomic_compare_exchange_strong_explicit((_Atomic(void *)*)obj, + (void **)expected, desired, + memory_order_relaxed, + memory_order_relaxed); +} + + // --- _Py_atomic_exchange --------------------------------------------------- static inline int diff --git a/Misc/NEWS.d/next/C_API/2026-08-20-15-21-46.gh-issue-155628.9QggKx.rst b/Misc/NEWS.d/next/C_API/2026-08-20-15-21-46.gh-issue-155628.9QggKx.rst index b904720a376790c..d2008e22320e4aa 100644 --- a/Misc/NEWS.d/next/C_API/2026-08-20-15-21-46.gh-issue-155628.9QggKx.rst +++ b/Misc/NEWS.d/next/C_API/2026-08-20-15-21-46.gh-issue-155628.9QggKx.rst @@ -1,5 +1,5 @@ -Add relaxed variants of the _Py_atomic_add functions. The existing add -operations are sequentially consistent, which is stronger (and on ARM, -measurably more expensive) than necessary for uses like statistics counters -and unique ID allocation, where the add must be atomic but does not need to -order surrounding memory accesses. +Add relaxed variants of the _Py_atomic_add and _Py_atomic_compare_exchange +functions. The existing operations are sequentially consistent, which is +stronger (and on ARM, measurably more expensive) than necessary for uses like +statistics counters and unique ID allocation, where the operation must be +atomic but does not need to order surrounding memory accesses. diff --git a/Modules/_testcapi/pyatomic.c b/Modules/_testcapi/pyatomic.c index 4f884af93592055..81947b55e9a81d2 100644 --- a/Modules/_testcapi/pyatomic.c +++ b/Modules/_testcapi/pyatomic.c @@ -74,6 +74,17 @@ test_atomic_compare_exchange_##suffix(PyObject *self, PyObject *obj) { \ assert(_Py_atomic_compare_exchange_##suffix(&x, &y, z) == 0); \ assert(x == z); \ assert(y == z); \ + x = (dtype)0; \ + y = (dtype)1; \ + assert(_Py_atomic_compare_exchange_##suffix##_relaxed(&x, &y, z) == 0); \ + assert(x == 0); \ + assert(y == 0); \ + assert(_Py_atomic_compare_exchange_##suffix##_relaxed(&x, &y, z) == 1); \ + assert(x == z); \ + assert(y == 0); \ + assert(_Py_atomic_compare_exchange_##suffix##_relaxed(&x, &y, z) == 0); \ + assert(x == z); \ + assert(y == z); \ Py_RETURN_NONE; \ } FOR_ALL_TYPES(IMPL_TEST_COMPARE_EXCHANGE) From 90a0b21df46cc9c126a6f0b6e8befab2eb9d0c81 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 9 Sep 2026 12:43:19 -0400 Subject: [PATCH 7/7] gh-155628: Use relaxed compare_exchange for gc young count --- Python/gc_free_threading.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index d88bc538d186594..164bb4baebab167 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2058,9 +2058,9 @@ record_deallocation(PyThreadState *tstate) if (new_count < 0) { new_count = 0; } - } while (!_Py_atomic_compare_exchange_int(&gcstate->young.count, - &count, - new_count)); + } while (!_Py_atomic_compare_exchange_int_relaxed(&gcstate->young.count, + &count, + new_count)); gc->alloc_count = 0; } }