From d63eccb09d7b2b1e0538b1d4aff3d9cadb5df648 Mon Sep 17 00:00:00 2001 From: dpdani Date: Wed, 21 Aug 2024 18:08:09 +0200 Subject: [PATCH 01/31] gh-117657: TSAN Fix races in `PyMember_Get` and `PyMember_Set`, for C extensions --- Lib/test/libregrtest/tsan.py | 1 + Lib/test/test_free_threading/test_slots.py | 228 +++++++++++++++++++++ 2 files changed, 229 insertions(+) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index dd18ae2584f5d8..7f36449d9060d7 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -25,6 +25,7 @@ 'test_threading_local', 'test_threadsignals', 'test_weakref', + 'test_free_threading.test_slots', ] diff --git a/Lib/test/test_free_threading/test_slots.py b/Lib/test/test_free_threading/test_slots.py index 758f74f54d0b56..710e74665aeb44 100644 --- a/Lib/test/test_free_threading/test_slots.py +++ b/Lib/test/test_free_threading/test_slots.py @@ -1,3 +1,4 @@ +import _testcapi import threading from test.support import threading_helper from unittest import TestCase @@ -41,3 +42,230 @@ def reader(): assert 0 <= eggs <= iters run_in_threads([writer, reader, reader, reader]) + + def test_T_BOOL(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + # different code paths for True and False + spam_old.T_BOOL = True + spam_new.T_BOOL = True + spam_old.T_BOOL = False + spam_new.T_BOOL = False + + def reader(): + for _ in range(1_000): + spam_old.T_BOOL + spam_new.T_BOOL + + run_in_threads([writer, reader]) + + def test_T_BYTE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_BYTE = 0 + spam_new.T_BYTE = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_BYTE + spam_new.T_BYTE + + run_in_threads([writer, reader]) + + def test_T_UBYTE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_UBYTE = 0 + spam_new.T_UBYTE = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_UBYTE + spam_new.T_UBYTE + + run_in_threads([writer, reader]) + + def test_T_SHORT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_SHORT = 0 + spam_new.T_SHORT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_SHORT + spam_new.T_SHORT + + run_in_threads([writer, reader]) + + def test_T_USHORT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_USHORT = 0 + spam_new.T_USHORT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_USHORT + spam_new.T_USHORT + + run_in_threads([writer, reader]) + + def test_T_INT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_INT = 0 + spam_new.T_INT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_INT + spam_new.T_INT + + run_in_threads([writer, reader]) + + def test_T_UINT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_UINT = 0 + spam_new.T_UINT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_UINT + spam_new.T_UINT + + run_in_threads([writer, reader]) + + def test_T_LONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_LONG = 0 + spam_new.T_LONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_LONG + spam_new.T_LONG + + run_in_threads([writer, reader]) + + def test_T_ULONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_ULONG = 0 + spam_new.T_ULONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_ULONG + spam_new.T_ULONG + + run_in_threads([writer, reader]) + + def test_T_PYSSIZET(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_PYSSIZET = 0 + spam_new.T_PYSSIZET = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_PYSSIZET + spam_new.T_PYSSIZET + + run_in_threads([writer, reader]) + + def test_T_FLOAT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_FLOAT = 0.0 + spam_new.T_FLOAT = 0.0 + + def reader(): + for _ in range(1_000): + spam_old.T_FLOAT + spam_new.T_FLOAT + + run_in_threads([writer, reader]) + + def test_T_DOUBLE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_DOUBLE = 0.0 + spam_new.T_DOUBLE = 0.0 + + def reader(): + for _ in range(1_000): + spam_old.T_DOUBLE + spam_new.T_DOUBLE + + run_in_threads([writer, reader]) + + def test_T_LONGLONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_LONGLONG = 0 + spam_new.T_LONGLONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_LONGLONG + spam_new.T_LONGLONG + + run_in_threads([writer, reader]) + + def test_T_ULONGLONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_ULONGLONG = 0 + spam_new.T_ULONGLONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_ULONGLONG + spam_new.T_ULONGLONG + + run_in_threads([writer, reader]) From c2d3d411d5d06b839db18255bcd26879aa4ac78f Mon Sep 17 00:00:00 2001 From: dpdani Date: Wed, 21 Aug 2024 22:56:02 +0200 Subject: [PATCH 02/31] exercise race in T_CHAR as well --- Lib/test/test_free_threading/test_slots.py | 16 ++++++++++++++++ Modules/_testcapi/structmember.c | 8 ++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_free_threading/test_slots.py b/Lib/test/test_free_threading/test_slots.py index 710e74665aeb44..a3b9f4b0175ae7 100644 --- a/Lib/test/test_free_threading/test_slots.py +++ b/Lib/test/test_free_threading/test_slots.py @@ -269,3 +269,19 @@ def reader(): spam_new.T_ULONGLONG run_in_threads([writer, reader]) + + def test_T_CHAR(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_CHAR = "c" + spam_new.T_CHAR = "c" + + def reader(): + for _ in range(1_000): + spam_old.T_CHAR + spam_new.T_CHAR + + run_in_threads([writer, reader]) diff --git a/Modules/_testcapi/structmember.c b/Modules/_testcapi/structmember.c index 096eaecd40855f..f136a25e316d46 100644 --- a/Modules/_testcapi/structmember.c +++ b/Modules/_testcapi/structmember.c @@ -22,6 +22,7 @@ typedef struct { char inplace_member[6]; long long longlong_member; unsigned long long ulonglong_member; + char char_member; } all_structmembers; typedef struct { @@ -46,6 +47,7 @@ static struct PyMemberDef test_members_newapi[] = { {"T_STRING_INPLACE", Py_T_STRING_INPLACE, offsetof(test_structmembers, structmembers.inplace_member), 0, NULL}, {"T_LONGLONG", Py_T_LONGLONG, offsetof(test_structmembers, structmembers.longlong_member), 0, NULL}, {"T_ULONGLONG", Py_T_ULONGLONG, offsetof(test_structmembers, structmembers.ulonglong_member), 0, NULL}, + {"T_CHAR", Py_T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, {NULL} }; @@ -56,7 +58,7 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "T_BOOL", "T_BYTE", "T_UBYTE", "T_SHORT", "T_USHORT", "T_INT", "T_UINT", "T_LONG", "T_ULONG", "T_PYSSIZET", "T_FLOAT", "T_DOUBLE", "T_STRING_INPLACE", - "T_LONGLONG", "T_ULONGLONG", + "T_LONGLONG", "T_ULONGLONG", "T_CHAR", NULL}; static const char fmt[] = "|bbBhHiIlknfds#LK"; test_structmembers *ob; @@ -82,7 +84,8 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) &ob->structmembers.double_member, &s, &string_len, &ob->structmembers.longlong_member, - &ob->structmembers.ulonglong_member)) + &ob->structmembers.ulonglong_member, + &ob->structmembers.char_member)) { Py_DECREF(ob); return NULL; @@ -132,6 +135,7 @@ static struct PyMemberDef test_members[] = { {"T_STRING_INPLACE", T_STRING_INPLACE, offsetof(test_structmembers, structmembers.inplace_member), 0, NULL}, {"T_LONGLONG", T_LONGLONG, offsetof(test_structmembers, structmembers.longlong_member), 0, NULL}, {"T_ULONGLONG", T_ULONGLONG, offsetof(test_structmembers, structmembers.ulonglong_member), 0, NULL}, + {"T_CHAR", Py_T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, {NULL} }; From f6ba4a33c60add7ed6667f07d950a5aa419daef6 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:13:14 +0200 Subject: [PATCH 03/31] signed char races --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 19 +++++++++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 7 ++++++- Python/structmember.c | 10 +++++----- 6 files changed, 60 insertions(+), 6 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 4ecef4f56edf42..b70d108b32c9eb 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -321,6 +321,9 @@ _Py_atomic_load_ptr(const void *obj); static inline int _Py_atomic_load_int_relaxed(const int *obj); +static inline char +_Py_atomic_load_char_relaxed(const char *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -481,6 +484,9 @@ _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); static inline void _Py_atomic_store_int_release(int *obj, int value); +static inline void +_Py_atomic_store_char_release(char *obj, char value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index ef09954d53ac1d..97a10ba5969464 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -306,6 +306,10 @@ static inline int _Py_atomic_load_int_relaxed(const int *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -508,6 +512,10 @@ static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_char_release(char *obj, char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 84da21bdcbff4f..d35ec380ea2a56 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -634,6 +634,12 @@ _Py_atomic_load_int_relaxed(const int *obj) return *(volatile int *)obj; } +static inline int +_Py_atomic_load_char_relaxed(const char *obj) +{ + return *(volatile char *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -964,6 +970,19 @@ _Py_atomic_store_int_release(int *obj, int value) #endif } +static inline void +_Py_atomic_store_char_release(char *obj, char value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(char volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int8); + __stlr8((unsigned __int8 volatile *)obj, (unsigned __int8)value); +#else +# error "no implementation of _Py_atomic_store_char_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 7c71e94c68f8e6..e5a76e1fcedc4f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -515,6 +515,14 @@ _Py_atomic_load_int_relaxed(const int *obj) memory_order_relaxed); } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(char)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -895,6 +903,14 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_char_release(char *obj, char value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(char)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index a1bb383bcd22e9..f78392411b256c 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -61,6 +61,10 @@ extern "C" { _Py_atomic_store_uint16_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ _Py_atomic_store_uint32_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_CHAR_RELEASE(value, new_value) \ + _Py_atomic_store_char_release(&value, new_value) +#define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ + _Py_atomic_load_char_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -68,7 +72,6 @@ extern "C" { #define FT_ATOMIC_LOAD_SSIZE(value) value #define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value -#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value @@ -85,6 +88,8 @@ extern "C" { #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value +#define FT_ATOMIC_STORE_CHAR_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index d5e7ab83093dc8..71dac80f8bcee2 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -34,10 +34,10 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) const char* addr = obj_addr + l->offset; switch (l->type) { case Py_T_BOOL: - v = PyBool_FromLong(*(char*)addr); + v = PyBool_FromLong(FT_ATOMIC_LOAD_CHAR_RELAXED(*(char*)addr)); break; case Py_T_BYTE: - v = PyLong_FromLong(*(char*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_CHAR_RELAXED(*(char*)addr)); break; case Py_T_UBYTE: v = PyLong_FromUnsignedLong(*(unsigned char*)addr); @@ -169,16 +169,16 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; } if (v == Py_True) - *(char*)addr = (char) 1; + FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, 1); else - *(char*)addr = (char) 0; + FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, 0); break; } case Py_T_BYTE:{ long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(char*)addr = (char)long_val; + FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, (char)long_val); /* XXX: For compatibility, only warn about truncations for now. */ if ((long_val > CHAR_MAX) || (long_val < CHAR_MIN)) From d9614b743743c1ebbd34f9b145dbdfb40b64d2c3 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:22:50 +0200 Subject: [PATCH 04/31] unsigned char race --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 13 +++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ Include/internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 6 files changed, 51 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index b70d108b32c9eb..60001c36c58036 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -324,6 +324,9 @@ _Py_atomic_load_int_relaxed(const int *obj); static inline char _Py_atomic_load_char_relaxed(const char *obj); +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -487,6 +490,9 @@ _Py_atomic_store_int_release(int *obj, int value); static inline void _Py_atomic_store_char_release(char *obj, char value); +static inline void +_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 97a10ba5969464..208d87e3df7db4 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -310,6 +310,10 @@ static inline char _Py_atomic_load_char_relaxed(const char *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -516,6 +520,10 @@ static inline void _Py_atomic_store_char_release(char *obj, char value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index d35ec380ea2a56..6ec8ae3546b4c3 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -983,6 +983,19 @@ _Py_atomic_store_char_release(char *obj, char value) #endif } +static inline void +_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(unsigned char volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int8); + __stlr8((unsigned __int8 volatile *)obj, (unsigned __int8)value); +#else +# error "no implementation of _Py_atomic_store_uchar_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index e5a76e1fcedc4f..988df67aa3e438 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -523,6 +523,14 @@ _Py_atomic_load_char_relaxed(const char *obj) memory_order_relaxed); } +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned char)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -911,6 +919,14 @@ _Py_atomic_store_char_release(char *obj, char value) memory_order_release); } +static inline void +_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned char)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index f78392411b256c..a96978767f2916 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -65,6 +65,10 @@ extern "C" { _Py_atomic_store_char_release(&value, new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ _Py_atomic_load_char_relaxed(&value) +#define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) \ + _Py_atomic_store_uchar_release(&value, new_value) +#define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) \ + _Py_atomic_load_uchar_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -90,6 +94,8 @@ extern "C" { #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value #define FT_ATOMIC_STORE_CHAR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value +#define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 71dac80f8bcee2..384516c1bd0baf 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -40,7 +40,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromLong(FT_ATOMIC_LOAD_CHAR_RELAXED(*(char*)addr)); break; case Py_T_UBYTE: - v = PyLong_FromUnsignedLong(*(unsigned char*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UCHAR_RELAXED(*(unsigned char*)addr)); break; case Py_T_SHORT: v = PyLong_FromLong(*(short*)addr); @@ -189,7 +189,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(unsigned char*)addr = (unsigned char)long_val; + FT_ATOMIC_STORE_UCHAR_RELEASE(*(unsigned char*)addr, (unsigned char)long_val); if ((long_val > UCHAR_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned char"); break; From 1659849c09f1b1efe1bb24dafe90fe6c7e3ae845 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:31:22 +0200 Subject: [PATCH 05/31] =?UTF-8?q?unsigned=20char=20race=20=F0=9F=91=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Include/cpython/pyatomic_msc.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 6ec8ae3546b4c3..fc24285abb0e07 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -640,6 +640,12 @@ _Py_atomic_load_char_relaxed(const char *obj) return *(volatile char *)obj; } +static inline int +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ + return *(volatile unsigned char *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { From 4cfa619f1dba51339e40da950dbfef37b07513eb Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:31:59 +0200 Subject: [PATCH 06/31] =?UTF-8?q?unsigned=20char=20race=20=F0=9F=91=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Include/cpython/pyatomic_msc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index fc24285abb0e07..7d578d3b7c7352 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -640,7 +640,7 @@ _Py_atomic_load_char_relaxed(const char *obj) return *(volatile char *)obj; } -static inline int +static inline unsigned char _Py_atomic_load_uchar_relaxed(const unsigned char *obj) { return *(volatile unsigned char *)obj; From 107a9f5d91f762455ac8f1f9ab90f92ed2d64f56 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:37:41 +0200 Subject: [PATCH 07/31] signed short race --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 19 +++++++++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 6 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 60001c36c58036..0d1726d6e6b01c 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -327,6 +327,9 @@ _Py_atomic_load_char_relaxed(const char *obj); static inline unsigned char _Py_atomic_load_uchar_relaxed(const unsigned char *obj); +static inline short +_Py_atomic_load_short_relaxed(const short *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -493,6 +496,9 @@ _Py_atomic_store_char_release(char *obj, char value); static inline void _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value); +static inline void +_Py_atomic_store_short_release(short *obj, short value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 208d87e3df7db4..edec79032b318e 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -314,6 +314,10 @@ static inline unsigned char _Py_atomic_load_uchar_relaxed(const unsigned char *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -524,6 +528,10 @@ static inline void _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_short_release(short *obj, short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 7d578d3b7c7352..0e0b4336ca5c9f 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -646,6 +646,12 @@ _Py_atomic_load_uchar_relaxed(const unsigned char *obj) return *(volatile unsigned char *)obj; } +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ + return *(volatile short *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -1002,6 +1008,19 @@ _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) #endif } +static inline void +_Py_atomic_store_short_release(short *obj, short value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(short volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); + __stlr8((unsigned __int16 volatile *)obj, (unsigned __int16)value); +#else +# error "no implementation of _Py_atomic_store_short_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 988df67aa3e438..79275aded376b7 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -531,6 +531,14 @@ _Py_atomic_load_uchar_relaxed(const unsigned char *obj) memory_order_relaxed); } +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(short)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -927,6 +935,14 @@ _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) memory_order_release); } +static inline void +_Py_atomic_store_short_release(short *obj, short value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(short)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index a96978767f2916..fe7e48eaab8e80 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -69,6 +69,10 @@ extern "C" { _Py_atomic_store_uchar_release(&value, new_value) #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) \ _Py_atomic_load_uchar_relaxed(&value) +#define FT_ATOMIC_STORE_SHORT_RELEASE(value, new_value) \ + _Py_atomic_store_short_release(&value, new_value) +#define FT_ATOMIC_LOAD_SHORT_RELAXED(value) \ + _Py_atomic_load_short_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -96,6 +100,8 @@ extern "C" { #define FT_ATOMIC_STORE_CHAR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value #define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_SHORT_RELAXED(value) value +#define FT_ATOMIC_STORE_SHORT_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 384516c1bd0baf..ef3cb6aebe74b4 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -43,7 +43,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UCHAR_RELAXED(*(unsigned char*)addr)); break; case Py_T_SHORT: - v = PyLong_FromLong(*(short*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_SHORT_RELAXED(*(short*)addr)); break; case Py_T_USHORT: v = PyLong_FromUnsignedLong(*(unsigned short*)addr); @@ -198,7 +198,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(short*)addr = (short)long_val; + FT_ATOMIC_STORE_SHORT_RELEASE(*(short*)addr, (short)long_val); if ((long_val > SHRT_MAX) || (long_val < SHRT_MIN)) WARN("Truncation of value to short"); break; From e20b9d158a63bded7663ebe3243920ef45d31c61 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:45:55 +0200 Subject: [PATCH 08/31] unsigned short race --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 19 +++++++++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 6 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 0d1726d6e6b01c..530759555447b3 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -330,6 +330,9 @@ _Py_atomic_load_uchar_relaxed(const unsigned char *obj); static inline short _Py_atomic_load_short_relaxed(const short *obj); +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -499,6 +502,9 @@ _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value); static inline void _Py_atomic_store_short_release(short *obj, short value); +static inline void +_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index edec79032b318e..cbe66ed88bdcc4 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -318,6 +318,10 @@ static inline short _Py_atomic_load_short_relaxed(const short *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -532,6 +536,10 @@ static inline void _Py_atomic_store_short_release(short *obj, short value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 0e0b4336ca5c9f..70c61118722c94 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -652,6 +652,12 @@ _Py_atomic_load_short_relaxed(const short *obj) return *(volatile short *)obj; } +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ + return *(volatile unsigned short *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -1021,6 +1027,19 @@ _Py_atomic_store_short_release(short *obj, short value) #endif } +static inline void +_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(short volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); + __stlr8((unsigned __int16 volatile *)obj, (unsigned __int16)value); +#else +# error "no implementation of _Py_atomic_store_ushort_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 79275aded376b7..3a0a1917602bea 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -539,6 +539,14 @@ _Py_atomic_load_short_relaxed(const short *obj) memory_order_relaxed); } +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned short)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -943,6 +951,14 @@ _Py_atomic_store_short_release(short *obj, short value) memory_order_release); } +static inline void +_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned short)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index fe7e48eaab8e80..b79f97c9f75e00 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -73,6 +73,10 @@ extern "C" { _Py_atomic_store_short_release(&value, new_value) #define FT_ATOMIC_LOAD_SHORT_RELAXED(value) \ _Py_atomic_load_short_relaxed(&value) +#define FT_ATOMIC_STORE_USHORT_RELEASE(value, new_value) \ + _Py_atomic_store_ushort_release(&value, new_value) +#define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ + _Py_atomic_load_ushort_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -102,6 +106,8 @@ extern "C" { #define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_SHORT_RELAXED(value) value #define FT_ATOMIC_STORE_SHORT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value +#define FT_ATOMIC_STORE_USHORT_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index ef3cb6aebe74b4..bd35889329d3e8 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -46,7 +46,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromLong(FT_ATOMIC_LOAD_SHORT_RELAXED(*(short*)addr)); break; case Py_T_USHORT: - v = PyLong_FromUnsignedLong(*(unsigned short*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_USHORT_RELAXED(*(unsigned short*)addr)); break; case Py_T_INT: v = PyLong_FromLong(*(int*)addr); @@ -207,7 +207,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(unsigned short*)addr = (unsigned short)long_val; + FT_ATOMIC_STORE_USHORT_RELEASE(*(unsigned short*)addr, (unsigned short)long_val); if ((long_val > USHRT_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned short"); break; From 86a0945b9e18bc5c02bffb2d84013123e73a7c1a Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:53:56 +0200 Subject: [PATCH 09/31] signed int race --- Include/internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index b79f97c9f75e00..1bd8f8bd3da117 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -77,6 +77,10 @@ extern "C" { _Py_atomic_store_ushort_release(&value, new_value) #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ _Py_atomic_load_ushort_relaxed(&value) +#define FT_ATOMIC_STORE_INT_RELEASE(value, new_value) \ + _Py_atomic_store_int_release(&value, new_value) +#define FT_ATOMIC_LOAD_INT_RELAXED(value) \ + _Py_atomic_load_int_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -108,6 +112,8 @@ extern "C" { #define FT_ATOMIC_STORE_SHORT_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value #define FT_ATOMIC_STORE_USHORT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT_RELAXED(value) value +#define FT_ATOMIC_STORE_INT_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index bd35889329d3e8..5e30325af45ccd 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -49,7 +49,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_USHORT_RELAXED(*(unsigned short*)addr)); break; case Py_T_INT: - v = PyLong_FromLong(*(int*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_INT_RELAXED(*(int*)addr)); break; case Py_T_UINT: v = PyLong_FromUnsignedLong(*(unsigned int*)addr); @@ -216,7 +216,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(int *)addr = (int)long_val; + FT_ATOMIC_STORE_INT_RELEASE(*(int *)addr, (int)long_val); if ((long_val > INT_MAX) || (long_val < INT_MIN)) WARN("Truncation of value to int"); break; From 1a07031cc679418810dfbb50f91e5e52084b59c3 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 16:01:22 +0200 Subject: [PATCH 10/31] who uses MSC anyways? --- Include/cpython/pyatomic_msc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 70c61118722c94..e064baa3e48837 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1021,7 +1021,7 @@ _Py_atomic_store_short_release(short *obj, short value) *(short volatile *)obj = value; #elif defined(_M_ARM64) _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); - __stlr8((unsigned __int16 volatile *)obj, (unsigned __int16)value); + __stlr16((unsigned __int16 volatile *)obj, (unsigned __int16)value); #else # error "no implementation of _Py_atomic_store_short_release" #endif @@ -1034,7 +1034,7 @@ _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) *(short volatile *)obj = value; #elif defined(_M_ARM64) _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); - __stlr8((unsigned __int16 volatile *)obj, (unsigned __int16)value); + __stlr16((unsigned __int16 volatile *)obj, (unsigned __int16)value); #else # error "no implementation of _Py_atomic_store_ushort_release" #endif From 16f61d8e88fbe2fd3403a5e8661249fb369198c1 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 16:47:38 +0200 Subject: [PATCH 11/31] unsigned int races --- Include/cpython/pyatomic.h | 3 +++ Include/cpython/pyatomic_gcc.h | 4 ++++ Include/cpython/pyatomic_msc.h | 13 +++++++++++++ Include/cpython/pyatomic_std.h | 8 ++++++++ Include/internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 6 +++--- 6 files changed, 37 insertions(+), 3 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 530759555447b3..081239dfbc403b 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -505,6 +505,9 @@ _Py_atomic_store_short_release(short *obj, short value); static inline void _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value); +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index cbe66ed88bdcc4..ea43d00732f9e3 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -540,6 +540,10 @@ static inline void _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index e064baa3e48837..85aba5e9ba8204 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1040,6 +1040,19 @@ _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) #endif } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(short volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 3a0a1917602bea..9621cc5b895a98 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -959,6 +959,14 @@ _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) memory_order_release); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 1bd8f8bd3da117..cae0ed107dff71 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -81,6 +81,10 @@ extern "C" { _Py_atomic_store_int_release(&value, new_value) #define FT_ATOMIC_LOAD_INT_RELAXED(value) \ _Py_atomic_load_int_relaxed(&value) +#define FT_ATOMIC_STORE_UINT_RELEASE(value, new_value) \ + _Py_atomic_store_uint_release(&value, new_value) +#define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ + _Py_atomic_load_uint_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -114,6 +118,8 @@ extern "C" { #define FT_ATOMIC_STORE_USHORT_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_INT_RELAXED(value) value #define FT_ATOMIC_STORE_INT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT_RELAXED(value) value +#define FT_ATOMIC_STORE_UINT_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 5e30325af45ccd..68d8dbdeaada8e 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -52,7 +52,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromLong(FT_ATOMIC_LOAD_INT_RELAXED(*(int*)addr)); break; case Py_T_UINT: - v = PyLong_FromUnsignedLong(*(unsigned int*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UINT_RELAXED(*(unsigned int*)addr)); break; case Py_T_LONG: v = PyLong_FromLong(*(long*)addr); @@ -234,7 +234,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned int *)addr = (unsigned int)(unsigned long)long_val; + FT_ATOMIC_STORE_UINT_RELEASE(*(unsigned int *)addr, (unsigned int)(unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -243,7 +243,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned int*)addr = (unsigned int)ulong_val; + FT_ATOMIC_STORE_UINT_RELEASE(*(unsigned int *)addr, (unsigned int)ulong_val); if (ulong_val > UINT_MAX) { WARN("Truncation of value to unsigned int"); } From cf25726af36048a7ef91826bd938c695328ffb48 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 16:54:34 +0200 Subject: [PATCH 12/31] =?UTF-8?q?=F0=9F=91=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Include/cpython/pyatomic_msc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 85aba5e9ba8204..047e4be793adc9 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1031,7 +1031,7 @@ static inline void _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) { #if defined(_M_X64) || defined(_M_IX86) - *(short volatile *)obj = value; + *(unsigned short volatile *)obj = value; #elif defined(_M_ARM64) _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); __stlr16((unsigned __int16 volatile *)obj, (unsigned __int16)value); @@ -1044,7 +1044,7 @@ static inline void _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) { #if defined(_M_X64) || defined(_M_IX86) - *(short volatile *)obj = value; + *(unsigned int volatile *)obj = value; #elif defined(_M_ARM64) _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); From d41132c879522d37d6564e3d3b44671b3cde4507 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 16:59:38 +0200 Subject: [PATCH 13/31] signed long race --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 19 +++++++++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 6 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 081239dfbc403b..ccf08fcee66944 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -333,6 +333,9 @@ _Py_atomic_load_short_relaxed(const short *obj); static inline unsigned short _Py_atomic_load_ushort_relaxed(const unsigned short *obj); +static inline long +_Py_atomic_load_long_relaxed(const long *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -508,6 +511,9 @@ _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value); static inline void _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); +static inline void +_Py_atomic_store_long_release(long *obj, long value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index ea43d00732f9e3..2602f9feeabf20 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -322,6 +322,10 @@ static inline unsigned short _Py_atomic_load_ushort_relaxed(const unsigned short *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -544,6 +548,10 @@ static inline void _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_long_release(long *obj, long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 047e4be793adc9..5453faf3cdaba4 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -658,6 +658,12 @@ _Py_atomic_load_ushort_relaxed(const unsigned short *obj) return *(volatile unsigned short *)obj; } +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + return *(volatile long *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -1053,6 +1059,19 @@ _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) #endif } +static inline void +_Py_atomic_store_long_release(long *obj, long value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(long volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_long_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 9621cc5b895a98..24e61688bd1b73 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -547,6 +547,14 @@ _Py_atomic_load_ushort_relaxed(const unsigned short *obj) memory_order_relaxed); } +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(long)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -967,6 +975,14 @@ _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) memory_order_release); } +static inline void +_Py_atomic_store_long_release(long *obj, long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(long)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index cae0ed107dff71..496063d99fb476 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -85,6 +85,10 @@ extern "C" { _Py_atomic_store_uint_release(&value, new_value) #define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ _Py_atomic_load_uint_relaxed(&value) +#define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) \ + _Py_atomic_store_long_release(&value, new_value) +#define FT_ATOMIC_LOAD_LONG_RELAXED(value) \ + _Py_atomic_load_long_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -120,6 +124,8 @@ extern "C" { #define FT_ATOMIC_STORE_INT_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UINT_RELAXED(value) value #define FT_ATOMIC_STORE_UINT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_LONG_RELAXED(value) value +#define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 68d8dbdeaada8e..2d67d59885dbd5 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -55,7 +55,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UINT_RELAXED(*(unsigned int*)addr)); break; case Py_T_LONG: - v = PyLong_FromLong(*(long*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_LONG_RELAXED(*(long*)addr)); break; case Py_T_ULONG: v = PyLong_FromUnsignedLong(*(unsigned long*)addr); @@ -251,7 +251,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case Py_T_LONG:{ - *(long*)addr = PyLong_AsLong(v); + FT_ATOMIC_STORE_LONG_RELEASE(*(long*)addr, PyLong_AsLong(v)); if ((*(long*)addr == -1) && PyErr_Occurred()) return -1; break; From 22126058eed48b6d88e402f1270fce276887052d Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 17:08:30 +0200 Subject: [PATCH 14/31] unsigned long race --- Include/internal/pycore_pyatomic_ft_wrappers.h | 3 +++ Python/structmember.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 496063d99fb476..204ce320d3fc10 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -89,6 +89,8 @@ extern "C" { _Py_atomic_store_long_release(&value, new_value) #define FT_ATOMIC_LOAD_LONG_RELAXED(value) \ _Py_atomic_load_long_relaxed(&value) +#define FT_ATOMIC_STORE_ULONG(value, new_value) \ + _Py_atomic_store_ulong(&value, new_value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -126,6 +128,7 @@ extern "C" { #define FT_ATOMIC_STORE_UINT_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_LONG_RELAXED(value) value #define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_ULONG(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 2d67d59885dbd5..827e48596dc3de 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -58,7 +58,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromLong(FT_ATOMIC_LOAD_LONG_RELAXED(*(long*)addr)); break; case Py_T_ULONG: - v = PyLong_FromUnsignedLong(*(unsigned long*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_ULONG_RELAXED(*(unsigned long*)addr)); break; case Py_T_PYSSIZET: v = PyLong_FromSsize_t(*(Py_ssize_t*)addr); @@ -269,7 +269,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned long *)addr = (unsigned long)long_val; + FT_ATOMIC_STORE_ULONG(*(unsigned long *)addr, (unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -278,7 +278,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned long*)addr = ulong_val; + FT_ATOMIC_STORE_ULONG(*(unsigned long *)addr, ulong_val); } break; } From 74616d648d62f257d6a858f19080e8104ed85630 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 17:13:40 +0200 Subject: [PATCH 15/31] ssizet race --- Include/internal/pycore_pyatomic_ft_wrappers.h | 3 +++ Python/structmember.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 204ce320d3fc10..3244f9e673cfff 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -91,6 +91,8 @@ extern "C" { _Py_atomic_load_long_relaxed(&value) #define FT_ATOMIC_STORE_ULONG(value, new_value) \ _Py_atomic_store_ulong(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) \ + _Py_atomic_store_ssize_release(&value, new_value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -129,6 +131,7 @@ extern "C" { #define FT_ATOMIC_LOAD_LONG_RELAXED(value) value #define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_ULONG(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 827e48596dc3de..565e63f571bcd0 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -61,7 +61,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_ULONG_RELAXED(*(unsigned long*)addr)); break; case Py_T_PYSSIZET: - v = PyLong_FromSsize_t(*(Py_ssize_t*)addr); + v = PyLong_FromSsize_t(FT_ATOMIC_LOAD_SSIZE_RELAXED(*(Py_ssize_t*)addr)); break; case Py_T_FLOAT: v = PyFloat_FromDouble((double)*(float*)addr); @@ -283,7 +283,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case Py_T_PYSSIZET:{ - *(Py_ssize_t*)addr = PyLong_AsSsize_t(v); + FT_ATOMIC_STORE_SSIZE_RELEASE(*(Py_ssize_t*)addr, PyLong_AsSsize_t(v)); if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; From 23d10c63bb40c227f1b7fa2bd82ff5d02d8728d9 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 17:31:39 +0200 Subject: [PATCH 16/31] float race --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 19 +++++++++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 6 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index ccf08fcee66944..f92b20a08b50dd 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -336,6 +336,9 @@ _Py_atomic_load_ushort_relaxed(const unsigned short *obj); static inline long _Py_atomic_load_long_relaxed(const long *obj); +static inline float +_Py_atomic_load_float_relaxed(const float *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -514,6 +517,9 @@ _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); static inline void _Py_atomic_store_long_release(long *obj, long value); +static inline void +_Py_atomic_store_float_release(float *obj, float value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 2602f9feeabf20..0132b09477ea90 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -326,6 +326,10 @@ static inline long _Py_atomic_load_long_relaxed(const long *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ float ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -552,6 +556,10 @@ static inline void _Py_atomic_store_long_release(long *obj, long value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_float_release(float *obj, float value) +{ __atomic_store(obj, &value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 5453faf3cdaba4..83c3e5ab106af5 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -664,6 +664,12 @@ _Py_atomic_load_long_relaxed(const long *obj) return *(volatile long *)obj; } +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ + return *(volatile float *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -1072,6 +1078,19 @@ _Py_atomic_store_long_release(long *obj, long value) #endif } +static inline void +_Py_atomic_store_float_release(float *obj, float value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(float volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr64((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_float_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 24e61688bd1b73..656783b7325173 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -555,6 +555,14 @@ _Py_atomic_load_long_relaxed(const long *obj) memory_order_relaxed); } +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(float)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -983,6 +991,14 @@ _Py_atomic_store_long_release(long *obj, long value) memory_order_release); } +static inline void +_Py_atomic_store_float_release(float *obj, float value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(float)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 3244f9e673cfff..489f8025cbac62 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -93,6 +93,10 @@ extern "C" { _Py_atomic_store_ulong(&value, new_value) #define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) \ _Py_atomic_store_ssize_release(&value, new_value) +#define FT_ATOMIC_STORE_FLOAT_RELEASE(value, new_value) \ + _Py_atomic_store_float_release(&value, new_value) +#define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) \ + _Py_atomic_load_float_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -132,6 +136,8 @@ extern "C" { #define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_ULONG(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) value +#define FT_ATOMIC_STORE_FLOAT_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 565e63f571bcd0..b4c509af12885f 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -64,7 +64,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromSsize_t(FT_ATOMIC_LOAD_SSIZE_RELAXED(*(Py_ssize_t*)addr)); break; case Py_T_FLOAT: - v = PyFloat_FromDouble((double)*(float*)addr); + v = PyFloat_FromDouble((double)FT_ATOMIC_LOAD_FLOAT_RELAXED(*(float*)addr)); break; case Py_T_DOUBLE: v = PyFloat_FromDouble(*(double*)addr); @@ -293,7 +293,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) double double_val = PyFloat_AsDouble(v); if ((double_val == -1) && PyErr_Occurred()) return -1; - *(float*)addr = (float)double_val; + FT_ATOMIC_STORE_FLOAT_RELEASE(*(float*)addr, (float)double_val); break; } case Py_T_DOUBLE: From ae37a8e68ca2c77ba1a528ab789a371815b0211e Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 17:38:28 +0200 Subject: [PATCH 17/31] double race --- Include/cpython/pyatomic.h | 7 +++++-- Include/cpython/pyatomic_gcc.h | 8 +++++++ Include/cpython/pyatomic_msc.h | 21 ++++++++++++++++++- Include/cpython/pyatomic_std.h | 16 ++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 6 files changed, 57 insertions(+), 5 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index f92b20a08b50dd..fbc30624ff3c1b 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -336,8 +336,8 @@ _Py_atomic_load_ushort_relaxed(const unsigned short *obj); static inline long _Py_atomic_load_long_relaxed(const long *obj); -static inline float -_Py_atomic_load_float_relaxed(const float *obj); +static inline double +_Py_atomic_load_double_relaxed(const double *obj); static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -520,6 +520,9 @@ _Py_atomic_store_long_release(long *obj, long value); static inline void _Py_atomic_store_float_release(float *obj, float value); +static inline void +_Py_atomic_store_double_release(double *obj, double value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 0132b09477ea90..744bd5cf811079 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -330,6 +330,10 @@ static inline float _Py_atomic_load_float_relaxed(const float *obj) { float ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ double ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -560,6 +564,10 @@ static inline void _Py_atomic_store_float_release(float *obj, float value) { __atomic_store(obj, &value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_double_release(double *obj, double value) +{ __atomic_store(obj, &value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 83c3e5ab106af5..93c57117f5f4e0 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -670,6 +670,12 @@ _Py_atomic_load_float_relaxed(const float *obj) return *(volatile float *)obj; } +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ + return *(volatile double *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -1085,12 +1091,25 @@ _Py_atomic_store_float_release(float *obj, float value) *(float volatile *)obj = value; #elif defined(_M_ARM64) _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); - __stlr64((unsigned __int32 volatile *)obj, (unsigned __int32)value); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); #else # error "no implementation of _Py_atomic_store_float_release" #endif } +static inline void +_Py_atomic_store_double_release(double *obj, double value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(double volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_double_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 656783b7325173..9f66ccfe41882b 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -563,6 +563,14 @@ _Py_atomic_load_float_relaxed(const float *obj) memory_order_relaxed); } +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(double)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -999,6 +1007,14 @@ _Py_atomic_store_float_release(float *obj, float value) memory_order_release); } +static inline void +_Py_atomic_store_double_release(double *obj, double value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(double)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 489f8025cbac62..5b3ea7e1a36885 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -97,6 +97,10 @@ extern "C" { _Py_atomic_store_float_release(&value, new_value) #define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) \ _Py_atomic_load_float_relaxed(&value) +#define FT_ATOMIC_STORE_DOUBLE_RELEASE(value, new_value) \ + _Py_atomic_store_double_release(&value, new_value) +#define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) \ + _Py_atomic_load_double_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -138,6 +142,8 @@ extern "C" { #define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) value #define FT_ATOMIC_STORE_FLOAT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) value +#define FT_ATOMIC_STORE_DOUBLE_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index b4c509af12885f..50558d867b79fb 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -67,7 +67,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyFloat_FromDouble((double)FT_ATOMIC_LOAD_FLOAT_RELAXED(*(float*)addr)); break; case Py_T_DOUBLE: - v = PyFloat_FromDouble(*(double*)addr); + v = PyFloat_FromDouble(FT_ATOMIC_LOAD_DOUBLE_RELAXED(*(double*)addr)); break; case Py_T_STRING: if (*(char**)addr == NULL) { @@ -297,7 +297,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case Py_T_DOUBLE: - *(double*)addr = PyFloat_AsDouble(v); + FT_ATOMIC_STORE_DOUBLE_RELEASE(*(double*)addr, PyFloat_AsDouble(v)); if ((*(double*)addr == -1) && PyErr_Occurred()) return -1; break; From bce02d0d02b28a533c8cef18a72bf41c5e50669b Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 17:48:09 +0200 Subject: [PATCH 18/31] T_CHAR --- Python/structmember.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Python/structmember.c b/Python/structmember.c index 50558d867b79fb..1b8f1ad140c8df 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -24,6 +24,7 @@ PyObject * PyMember_GetOne(const char *obj_addr, PyMemberDef *l) { PyObject *v; + char value; if (l->flags & Py_RELATIVE_OFFSET) { PyErr_SetString( PyExc_SystemError, @@ -80,7 +81,9 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyUnicode_FromString((char*)addr); break; case Py_T_CHAR: - v = PyUnicode_FromStringAndSize((char*)addr, 1); + // case ...: char value = ... requires a C23-compatible compiler + value = FT_ATOMIC_LOAD_CHAR_RELAXED(*(char *) addr); + v = PyUnicode_FromStringAndSize(&value, 1); break; case _Py_T_OBJECT: v = *(PyObject **)addr; @@ -318,7 +321,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_BadArgument(); return -1; } - *(char*)addr = string[0]; + FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, string[0]); break; } case Py_T_STRING: From afe25045275d8fbaf2e5d046a767cce070a28daf Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 18:02:33 +0200 Subject: [PATCH 19/31] signed long long race --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 19 +++++++++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 6 +++--- 6 files changed, 58 insertions(+), 3 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index fbc30624ff3c1b..5f862ef318879a 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -339,6 +339,9 @@ _Py_atomic_load_long_relaxed(const long *obj); static inline double _Py_atomic_load_double_relaxed(const double *obj); +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -523,6 +526,9 @@ _Py_atomic_store_float_release(float *obj, float value); static inline void _Py_atomic_store_double_release(double *obj, double value); +static inline void +_Py_atomic_store_llong_release(long long *obj, long long value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 744bd5cf811079..b856dde6a4f8c7 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -390,6 +390,10 @@ static inline unsigned long long _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + // --- _Py_atomic_store ------------------------------------------------------ @@ -568,6 +572,10 @@ static inline void _Py_atomic_store_double_release(double *obj, double value) { __atomic_store(obj, &value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_llong_release(long long *obj, long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 93c57117f5f4e0..ed87d6faa26cf6 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -760,6 +760,12 @@ _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) return *(volatile unsigned long long *)obj; } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ + return *(volatile long long *)obj; +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -1110,6 +1116,19 @@ _Py_atomic_store_double_release(double *obj, double value) #endif } +static inline void +_Py_atomic_store_llong_release(long long *obj, long long value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(long long volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int128); + __stlr128((unsigned __int128 volatile *)obj, (unsigned __int128)value); +#else +# error "no implementation of _Py_atomic_store_llong_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 9f66ccfe41882b..8d11633a69335f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -683,6 +683,14 @@ _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) memory_order_relaxed); } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(long long)*)obj, + memory_order_relaxed); +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -1015,6 +1023,14 @@ _Py_atomic_store_double_release(double *obj, double value) memory_order_release); } +static inline void +_Py_atomic_store_llong_release(long long *obj, long long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(long long)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 5b3ea7e1a36885..55193315ff5145 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -101,6 +101,10 @@ extern "C" { _Py_atomic_store_double_release(&value, new_value) #define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) \ _Py_atomic_load_double_relaxed(&value) +#define FT_ATOMIC_STORE_LLONG_RELEASE(value, new_value) \ + _Py_atomic_store_llong_release(&value, new_value) +#define FT_ATOMIC_LOAD_LLONG_RELAXED(value) \ + _Py_atomic_load_llong_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -144,6 +148,8 @@ extern "C" { #define FT_ATOMIC_STORE_FLOAT_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) value #define FT_ATOMIC_STORE_DOUBLE_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_LLONG_RELAXED(value) value +#define FT_ATOMIC_STORE_LLONG_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 1b8f1ad140c8df..6455f3587dabf2 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -107,7 +107,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) #endif break; case Py_T_LONGLONG: - v = PyLong_FromLongLong(*(long long *)addr); + v = PyLong_FromLongLong(FT_ATOMIC_LOAD_LLONG_RELAXED(*(long long *)addr)); break; case Py_T_ULONGLONG: v = PyLong_FromUnsignedLongLong(*(unsigned long long *)addr); @@ -329,8 +329,8 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_SetString(PyExc_TypeError, "readonly attribute"); return -1; case Py_T_LONGLONG:{ - long long value; - *(long long*)addr = value = PyLong_AsLongLong(v); + long long value = PyLong_AsLongLong(v); + FT_ATOMIC_STORE_LLONG_RELEASE(*(long long*)addr, value); if ((value == -1) && PyErr_Occurred()) return -1; break; From 3dfaab8865bae33cb56144466a4a1844febcde51 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 18:09:07 +0200 Subject: [PATCH 20/31] unsigned long long race --- Include/cpython/pyatomic.h | 4 ++++ Include/cpython/pyatomic_gcc.h | 5 +++++ Include/cpython/pyatomic_msc.h | 13 +++++++++++++ Include/cpython/pyatomic_std.h | 9 +++++++++ Include/internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 6 +++--- 6 files changed, 40 insertions(+), 3 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 5f862ef318879a..f5ccae1f207556 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -529,6 +529,10 @@ _Py_atomic_store_double_release(double *obj, double value); static inline void _Py_atomic_store_llong_release(long long *obj, long long value); +static inline void +_Py_atomic_store_ullong_release(unsigned long long *obj, + unsigned long long value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index b856dde6a4f8c7..8111611d8657f2 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -576,6 +576,11 @@ static inline void _Py_atomic_store_llong_release(long long *obj, long long value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_ullong_release(unsigned long long *obj, + unsigned long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index ed87d6faa26cf6..fa58a031d65282 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1129,6 +1129,19 @@ _Py_atomic_store_llong_release(long long *obj, long long value) #endif } +static inline void +_Py_atomic_store_ullong_release(unsigned long long *obj, unsigned long long value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(unsigned long long volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int128); + __stlr128((unsigned __int128 volatile *)obj, (unsigned __int128)value); +#else +# error "no implementation of _Py_atomic_store_ullong_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 8d11633a69335f..7e0018c901d62f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1031,6 +1031,15 @@ _Py_atomic_store_llong_release(long long *obj, long long value) memory_order_release); } +static inline void +_Py_atomic_store_ullong_release(unsigned long long *obj, + unsigned long long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned long long)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 55193315ff5145..d9703cb3274786 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -105,6 +105,10 @@ extern "C" { _Py_atomic_store_llong_release(&value, new_value) #define FT_ATOMIC_LOAD_LLONG_RELAXED(value) \ _Py_atomic_load_llong_relaxed(&value) +#define FT_ATOMIC_STORE_ULLONG_RELEASE(value, new_value) \ + _Py_atomic_store_ullong_release(&value, new_value) +#define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) \ + _Py_atomic_load_ullong_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -150,6 +154,8 @@ extern "C" { #define FT_ATOMIC_STORE_DOUBLE_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_LLONG_RELAXED(value) value #define FT_ATOMIC_STORE_LLONG_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value +#define FT_ATOMIC_STORE_ULLONG_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 6455f3587dabf2..3484d397d16021 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -110,7 +110,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromLongLong(FT_ATOMIC_LOAD_LLONG_RELAXED(*(long long *)addr)); break; case Py_T_ULONGLONG: - v = PyLong_FromUnsignedLongLong(*(unsigned long long *)addr); + v = PyLong_FromUnsignedLongLong(FT_ATOMIC_LOAD_ULLONG_RELAXED(*(unsigned long long *)addr)); break; case _Py_T_NONE: // doesn't require free-threading code path @@ -346,7 +346,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned long long *)addr = (unsigned long long)(long long)long_val; + FT_ATOMIC_STORE_ULLONG_RELEASE(*(unsigned long long *)addr, (unsigned long long)(long long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -355,7 +355,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulonglong_val == (unsigned long long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned long long*)addr = ulonglong_val; + FT_ATOMIC_STORE_ULLONG_RELEASE(*(unsigned long long *)addr, ulonglong_val); } break; } From 50a7bb61adc5c6f00f627514f6729f20289778c7 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 18:18:59 +0200 Subject: [PATCH 21/31] fix return type --- Include/cpython/pyatomic_msc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index fa58a031d65282..fdffcfe75118d1 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -634,7 +634,7 @@ _Py_atomic_load_int_relaxed(const int *obj) return *(volatile int *)obj; } -static inline int +static inline char _Py_atomic_load_char_relaxed(const char *obj) { return *(volatile char *)obj; From 416e0bb8f83d081e36a8a2e5da24968191714ef1 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 18:37:07 +0200 Subject: [PATCH 22/31] arm64 does not have 128-bit integers, I suppose --- Include/cpython/pyatomic_msc.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index fdffcfe75118d1..58dfdaaaf315ec 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1122,8 +1122,8 @@ _Py_atomic_store_llong_release(long long *obj, long long value) #if defined(_M_X64) || defined(_M_IX86) *(long long volatile *)obj = value; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int128); - __stlr128((unsigned __int128 volatile *)obj, (unsigned __int128)value); + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); #else # error "no implementation of _Py_atomic_store_llong_release" #endif @@ -1135,8 +1135,8 @@ _Py_atomic_store_ullong_release(unsigned long long *obj, unsigned long long valu #if defined(_M_X64) || defined(_M_IX86) *(unsigned long long volatile *)obj = value; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int128); - __stlr128((unsigned __int128 volatile *)obj, (unsigned __int128)value); + _Py_atomic_ASSERT_ARG_TYPE(__int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); #else # error "no implementation of _Py_atomic_store_ullong_release" #endif From 4bd69275a52bcde0863dd936cc3b14aa0aca2fbb Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 18:43:56 +0200 Subject: [PATCH 23/31] at this point I'm just making guesses --- Include/cpython/pyatomic_msc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 58dfdaaaf315ec..cf86c59cca9366 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1083,8 +1083,8 @@ _Py_atomic_store_long_release(long *obj, long value) #if defined(_M_X64) || defined(_M_IX86) *(long volatile *)obj = value; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); - __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); #else # error "no implementation of _Py_atomic_store_long_release" #endif From a4c094b1a42a871d49fdd9b2cf8d3ab0afbee096 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 29 Aug 2024 10:54:35 +0200 Subject: [PATCH 24/31] scope --- Python/structmember.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Python/structmember.c b/Python/structmember.c index 3484d397d16021..0f089c3a372caa 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -24,7 +24,6 @@ PyObject * PyMember_GetOne(const char *obj_addr, PyMemberDef *l) { PyObject *v; - char value; if (l->flags & Py_RELATIVE_OFFSET) { PyErr_SetString( PyExc_SystemError, @@ -80,11 +79,11 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) case Py_T_STRING_INPLACE: v = PyUnicode_FromString((char*)addr); break; - case Py_T_CHAR: - // case ...: char value = ... requires a C23-compatible compiler - value = FT_ATOMIC_LOAD_CHAR_RELAXED(*(char *) addr); + case Py_T_CHAR: { + char value; v = PyUnicode_FromStringAndSize(&value, 1); break; + } case _Py_T_OBJECT: v = *(PyObject **)addr; if (v == NULL) From 267f99572935b8a8d215de9d87741de66fa2cf62 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 29 Aug 2024 11:38:24 +0200 Subject: [PATCH 25/31] must 0-initialize --- Python/structmember.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/structmember.c b/Python/structmember.c index 0f089c3a372caa..d6de425cf4978b 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -80,7 +80,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyUnicode_FromString((char*)addr); break; case Py_T_CHAR: { - char value; + char value = 0; v = PyUnicode_FromStringAndSize(&value, 1); break; } From c006e1d465d727d38187dac2f62a1d06ada2b5ad Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 29 Aug 2024 12:28:27 +0200 Subject: [PATCH 26/31] FT_ATOMIC_STORE_CHAR_RELEASE -> FT_ATOMIC_STORE_CHAR_RELAXED --- Include/cpython/pyatomic.h | 6 +++--- Include/cpython/pyatomic_gcc.h | 8 ++++---- Include/cpython/pyatomic_msc.h | 19 ++++++------------- Include/cpython/pyatomic_std.h | 16 ++++++++-------- .../internal/pycore_pyatomic_ft_wrappers.h | 4 ++-- Python/structmember.c | 8 ++++---- 6 files changed, 27 insertions(+), 34 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index f5ccae1f207556..96cc06ce3a61bc 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -479,6 +479,9 @@ static inline void _Py_atomic_store_ullong_relaxed(unsigned long long *obj, unsigned long long value); +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value); + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -502,9 +505,6 @@ _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); static inline void _Py_atomic_store_int_release(int *obj, int value); -static inline void -_Py_atomic_store_char_release(char *obj, char value); - static inline void _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 8111611d8657f2..b11b8a2368d44c 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -517,6 +517,10 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, unsigned long long value) { __atomic_store_n(obj, value, __ATOMIC_RELAXED); } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -540,10 +544,6 @@ static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } -static inline void -_Py_atomic_store_char_release(char *obj, char value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } - static inline void _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index cf86c59cca9366..51708b97fc9479 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -947,6 +947,12 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, *(volatile unsigned long long *)obj = value; } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ + *(volatile char *)obj = value; +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -1012,19 +1018,6 @@ _Py_atomic_store_int_release(int *obj, int value) #endif } -static inline void -_Py_atomic_store_char_release(char *obj, char value) -{ -#if defined(_M_X64) || defined(_M_IX86) - *(char volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int8); - __stlr8((unsigned __int8 volatile *)obj, (unsigned __int8)value); -#else -# error "no implementation of _Py_atomic_store_char_release" -#endif -} - static inline void _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 7e0018c901d62f..c2a411466327ac 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -916,6 +916,14 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, memory_order_relaxed); } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(char)*)obj, value, + memory_order_relaxed); +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -959,14 +967,6 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } -static inline void -_Py_atomic_store_char_release(char *obj, char value) -{ - _Py_USING_STD; - atomic_store_explicit((_Atomic(char)*)obj, value, - memory_order_release); -} - static inline void _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index d9703cb3274786..914dba0b5f6792 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -61,8 +61,8 @@ extern "C" { _Py_atomic_store_uint16_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ _Py_atomic_store_uint32_relaxed(&value, new_value) -#define FT_ATOMIC_STORE_CHAR_RELEASE(value, new_value) \ - _Py_atomic_store_char_release(&value, new_value) +#define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \ + _Py_atomic_store_char_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ _Py_atomic_load_char_relaxed(&value) #define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) \ diff --git a/Python/structmember.c b/Python/structmember.c index d6de425cf4978b..5fbde435031631 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -171,16 +171,16 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; } if (v == Py_True) - FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, 1); + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, 1); else - FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, 0); + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, 0); break; } case Py_T_BYTE:{ long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, (char)long_val); + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, (char)long_val); /* XXX: For compatibility, only warn about truncations for now. */ if ((long_val > CHAR_MAX) || (long_val < CHAR_MIN)) @@ -320,7 +320,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_BadArgument(); return -1; } - FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, string[0]); + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, string[0]); break; } case Py_T_STRING: From 488dea52797332afd539797cbce8d93c17d3bd08 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 29 Aug 2024 14:25:07 +0200 Subject: [PATCH 27/31] remaining release -> relaxed --- Include/cpython/pyatomic.h | 47 +++--- Include/cpython/pyatomic_gcc.h | 63 ++++---- Include/cpython/pyatomic_msc.h | 153 +++++------------- Include/cpython/pyatomic_std.h | 93 +++++------ .../internal/pycore_pyatomic_ft_wrappers.h | 48 +++--- Python/structmember.c | 30 ++-- 6 files changed, 170 insertions(+), 264 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 96cc06ce3a61bc..6d106c1b499c69 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -482,56 +482,49 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, static inline void _Py_atomic_store_char_relaxed(char *obj, char value); - -// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ - -// Loads `*obj` (acquire operation) -static inline void * -_Py_atomic_load_ptr_acquire(const void *obj); - -static inline uintptr_t -_Py_atomic_load_uintptr_acquire(const uintptr_t *obj); - -// Stores `*obj = value` (release operation) static inline void -_Py_atomic_store_ptr_release(void *obj, void *value); +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value); static inline void -_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); +_Py_atomic_store_short_relaxed(short *obj, short value); static inline void -_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value); static inline void -_Py_atomic_store_int_release(int *obj, int value); +_Py_atomic_store_long_relaxed(long *obj, long value); static inline void -_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value); +_Py_atomic_store_float_relaxed(float *obj, float value); static inline void -_Py_atomic_store_short_release(short *obj, short value); +_Py_atomic_store_double_relaxed(double *obj, double value); static inline void -_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value); +_Py_atomic_store_llong_relaxed(long long *obj, long long value); -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); -static inline void -_Py_atomic_store_long_release(long *obj, long value); +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +// Loads `*obj` (acquire operation) +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj); +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj); + +// Stores `*obj = value` (release operation) static inline void -_Py_atomic_store_float_release(float *obj, float value); +_Py_atomic_store_ptr_release(void *obj, void *value); static inline void -_Py_atomic_store_double_release(double *obj, double value); +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); static inline void -_Py_atomic_store_llong_release(long long *obj, long long value); +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); static inline void -_Py_atomic_store_ullong_release(unsigned long long *obj, - unsigned long long value); +_Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index b11b8a2368d44c..b179e4c9a185a9 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -521,64 +521,55 @@ static inline void _Py_atomic_store_char_relaxed(char *obj, char value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } - -// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ - -static inline void * -_Py_atomic_load_ptr_acquire(const void *obj) -{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_ACQUIRE); } - -static inline uintptr_t -_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) -{ return (uintptr_t)__atomic_load_n(obj, __ATOMIC_ACQUIRE); } - static inline void -_Py_atomic_store_ptr_release(void *obj, void *value) -{ __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } static inline void -_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_short_relaxed(short *obj, short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } static inline void -_Py_atomic_store_int_release(int *obj, int value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } static inline void -_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_long_relaxed(long *obj, long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } static inline void -_Py_atomic_store_short_release(short *obj, short value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_float_relaxed(float *obj, float value) +{ __atomic_store(obj, &value, __ATOMIC_RELAXED); } static inline void -_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_double_relaxed(double *obj, double value) +{ __atomic_store(obj, &value, __ATOMIC_RELAXED); } static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_llong_relaxed(long long *obj, long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } -static inline void -_Py_atomic_store_long_release(long *obj, long value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } -static inline void -_Py_atomic_store_float_release(float *obj, float value) -{ __atomic_store(obj, &value, __ATOMIC_RELEASE); } +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_ACQUIRE); } + +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ return (uintptr_t)__atomic_load_n(obj, __ATOMIC_ACQUIRE); } static inline void -_Py_atomic_store_double_release(double *obj, double value) -{ __atomic_store(obj, &value, __ATOMIC_RELEASE); } +_Py_atomic_store_ptr_release(void *obj, void *value) +{ __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); } static inline void -_Py_atomic_store_llong_release(long long *obj, long long value) +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } static inline void -_Py_atomic_store_ullong_release(unsigned long long *obj, - unsigned long long value) +_Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } static inline void diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 51708b97fc9479..a7600114a64aef 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -953,185 +953,116 @@ _Py_atomic_store_char_relaxed(char *obj, char value) *(volatile char *)obj = value; } - -// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ - -static inline void * -_Py_atomic_load_ptr_acquire(const void *obj) -{ -#if defined(_M_X64) || defined(_M_IX86) - return *(void * volatile *)obj; -#elif defined(_M_ARM64) - return (void *)__ldar64((unsigned __int64 volatile *)obj); -#else -# error "no implementation of _Py_atomic_load_ptr_acquire" -#endif -} - -static inline uintptr_t -_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) { -#if defined(_M_X64) || defined(_M_IX86) - return *(uintptr_t volatile *)obj; -#elif defined(_M_ARM64) - return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj); -#else -# error "no implementation of _Py_atomic_load_uintptr_acquire" -#endif + *(volatile unsigned char*)obj = value; } static inline void -_Py_atomic_store_ptr_release(void *obj, void *value) +_Py_atomic_store_short_relaxed(short *obj, short value) { -#if defined(_M_X64) || defined(_M_IX86) - *(void * volatile *)obj = value; -#elif defined(_M_ARM64) - __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value); -#else -# error "no implementation of _Py_atomic_store_ptr_release" -#endif + *(volatile short*)obj = value; } static inline void -_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) { -#if defined(_M_X64) || defined(_M_IX86) - *(uintptr_t volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); - __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); -#else -# error "no implementation of _Py_atomic_store_uintptr_release" -#endif + *(volatile unsigned short*)obj = value; } static inline void -_Py_atomic_store_int_release(int *obj, int value) +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) { -#if defined(_M_X64) || defined(_M_IX86) - *(int volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); - __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); -#else -# error "no implementation of _Py_atomic_store_int_release" -#endif + *(volatile unsigned int*)obj = value; } static inline void -_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) +_Py_atomic_store_long_relaxed(long *obj, long value) { -#if defined(_M_X64) || defined(_M_IX86) - *(unsigned char volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int8); - __stlr8((unsigned __int8 volatile *)obj, (unsigned __int8)value); -#else -# error "no implementation of _Py_atomic_store_uchar_release" -#endif + *(volatile long *)obj = value; } static inline void -_Py_atomic_store_short_release(short *obj, short value) +_Py_atomic_store_float_relaxed(float *obj, float value) { -#if defined(_M_X64) || defined(_M_IX86) - *(short volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); - __stlr16((unsigned __int16 volatile *)obj, (unsigned __int16)value); -#else -# error "no implementation of _Py_atomic_store_short_release" -#endif + *(volatile float *)obj = value; } static inline void -_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) +_Py_atomic_store_double_relaxed(double *obj, double value) { -#if defined(_M_X64) || defined(_M_IX86) - *(unsigned short volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); - __stlr16((unsigned __int16 volatile *)obj, (unsigned __int16)value); -#else -# error "no implementation of _Py_atomic_store_ushort_release" -#endif + *(volatile double *)obj = value; } static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +_Py_atomic_store_llong_relaxed(long long *obj, long long value) { -#if defined(_M_X64) || defined(_M_IX86) - *(unsigned int volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); - __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); -#else -# error "no implementation of _Py_atomic_store_uint_release" -#endif + *(volatile long long*)obj = value; } -static inline void -_Py_atomic_store_long_release(long *obj, long value) + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) { #if defined(_M_X64) || defined(_M_IX86) - *(long volatile *)obj = value; + return *(void * volatile *)obj; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); - __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); + return (void *)__ldar64((unsigned __int64 volatile *)obj); #else -# error "no implementation of _Py_atomic_store_long_release" +# error "no implementation of _Py_atomic_load_ptr_acquire" #endif } -static inline void -_Py_atomic_store_float_release(float *obj, float value) +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) { #if defined(_M_X64) || defined(_M_IX86) - *(float volatile *)obj = value; + return *(uintptr_t volatile *)obj; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); - __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); + return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj); #else -# error "no implementation of _Py_atomic_store_float_release" +# error "no implementation of _Py_atomic_load_uintptr_acquire" #endif } static inline void -_Py_atomic_store_double_release(double *obj, double value) +_Py_atomic_store_ptr_release(void *obj, void *value) { #if defined(_M_X64) || defined(_M_IX86) - *(double volatile *)obj = value; + *(void * volatile *)obj = value; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); - __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); + __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value); #else -# error "no implementation of _Py_atomic_store_double_release" +# error "no implementation of _Py_atomic_store_ptr_release" #endif } static inline void -_Py_atomic_store_llong_release(long long *obj, long long value) +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) { #if defined(_M_X64) || defined(_M_IX86) - *(long long volatile *)obj = value; + *(uintptr_t volatile *)obj = value; #elif defined(_M_ARM64) _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); #else -# error "no implementation of _Py_atomic_store_llong_release" +# error "no implementation of _Py_atomic_store_uintptr_release" #endif } static inline void -_Py_atomic_store_ullong_release(unsigned long long *obj, unsigned long long value) +_Py_atomic_store_int_release(int *obj, int value) { #if defined(_M_X64) || defined(_M_IX86) - *(unsigned long long volatile *)obj = value; + *(int volatile *)obj = value; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(__int64); - __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); #else -# error "no implementation of _Py_atomic_store_ullong_release" +# error "no implementation of _Py_atomic_store_int_release" #endif } diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index c2a411466327ac..69a8b9e615ea5f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -924,119 +924,110 @@ _Py_atomic_store_char_relaxed(char *obj, char value) memory_order_relaxed); } - -// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ - -static inline void * -_Py_atomic_load_ptr_acquire(const void *obj) -{ - _Py_USING_STD; - return atomic_load_explicit((const _Atomic(void*)*)obj, - memory_order_acquire); -} - -static inline uintptr_t -_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) { _Py_USING_STD; - return atomic_load_explicit((const _Atomic(uintptr_t)*)obj, - memory_order_acquire); + atomic_store_explicit((_Atomic(unsigned char)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_ptr_release(void *obj, void *value) +_Py_atomic_store_short_relaxed(short *obj, short value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(void*)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(short)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(uintptr_t)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(unsigned short)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_int_release(int *obj, int value) +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(int)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) +_Py_atomic_store_long_relaxed(long *obj, long value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned char)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(long)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_short_release(short *obj, short value) +_Py_atomic_store_float_relaxed(float *obj, float value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(short)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(float)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) +_Py_atomic_store_double_relaxed(double *obj, double value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned short)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(double)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +_Py_atomic_store_llong_relaxed(long long *obj, long long value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned int)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(long long)*)obj, value, + memory_order_relaxed); } -static inline void -_Py_atomic_store_long_release(long *obj, long value) + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) { _Py_USING_STD; - atomic_store_explicit((_Atomic(long)*)obj, value, - memory_order_release); + return atomic_load_explicit((const _Atomic(void*)*)obj, + memory_order_acquire); } -static inline void -_Py_atomic_store_float_release(float *obj, float value) +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) { _Py_USING_STD; - atomic_store_explicit((_Atomic(float)*)obj, value, - memory_order_release); + return atomic_load_explicit((const _Atomic(uintptr_t)*)obj, + memory_order_acquire); } static inline void -_Py_atomic_store_double_release(double *obj, double value) +_Py_atomic_store_ptr_release(void *obj, void *value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(double)*)obj, value, + atomic_store_explicit((_Atomic(void*)*)obj, value, memory_order_release); } static inline void -_Py_atomic_store_llong_release(long long *obj, long long value) +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(long long)*)obj, value, + atomic_store_explicit((_Atomic(uintptr_t)*)obj, value, memory_order_release); } static inline void -_Py_atomic_store_ullong_release(unsigned long long *obj, - unsigned long long value) +_Py_atomic_store_int_release(int *obj, int value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned long long)*)obj, value, + atomic_store_explicit((_Atomic(int)*)obj, value, memory_order_release); } diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 914dba0b5f6792..f7b81f757b33e8 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -65,48 +65,48 @@ extern "C" { _Py_atomic_store_char_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ _Py_atomic_load_char_relaxed(&value) -#define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) \ - _Py_atomic_store_uchar_release(&value, new_value) +#define FT_ATOMIC_STORE_UCHAR_RELAXED(value, new_value) \ + _Py_atomic_store_uchar_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) \ _Py_atomic_load_uchar_relaxed(&value) -#define FT_ATOMIC_STORE_SHORT_RELEASE(value, new_value) \ - _Py_atomic_store_short_release(&value, new_value) +#define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) \ + _Py_atomic_store_short_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_SHORT_RELAXED(value) \ _Py_atomic_load_short_relaxed(&value) -#define FT_ATOMIC_STORE_USHORT_RELEASE(value, new_value) \ - _Py_atomic_store_ushort_release(&value, new_value) +#define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) \ + _Py_atomic_store_ushort_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ _Py_atomic_load_ushort_relaxed(&value) -#define FT_ATOMIC_STORE_INT_RELEASE(value, new_value) \ - _Py_atomic_store_int_release(&value, new_value) +#define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) \ + _Py_atomic_store_int_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_INT_RELAXED(value) \ _Py_atomic_load_int_relaxed(&value) -#define FT_ATOMIC_STORE_UINT_RELEASE(value, new_value) \ - _Py_atomic_store_uint_release(&value, new_value) +#define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) \ + _Py_atomic_store_uint_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ _Py_atomic_load_uint_relaxed(&value) -#define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) \ - _Py_atomic_store_long_release(&value, new_value) +#define FT_ATOMIC_STORE_LONG_RELAXED(value, new_value) \ + _Py_atomic_store_long_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_LONG_RELAXED(value) \ _Py_atomic_load_long_relaxed(&value) -#define FT_ATOMIC_STORE_ULONG(value, new_value) \ - _Py_atomic_store_ulong(&value, new_value) -#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) \ - _Py_atomic_store_ssize_release(&value, new_value) -#define FT_ATOMIC_STORE_FLOAT_RELEASE(value, new_value) \ - _Py_atomic_store_float_release(&value, new_value) +#define FT_ATOMIC_STORE_ULONG_RELAXED(value, new_value) \ + _Py_atomic_store_ulong_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ + _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) \ + _Py_atomic_store_float_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) \ _Py_atomic_load_float_relaxed(&value) -#define FT_ATOMIC_STORE_DOUBLE_RELEASE(value, new_value) \ - _Py_atomic_store_double_release(&value, new_value) +#define FT_ATOMIC_STORE_DOUBLE_RELAXED(value, new_value) \ + _Py_atomic_store_double_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) \ _Py_atomic_load_double_relaxed(&value) -#define FT_ATOMIC_STORE_LLONG_RELEASE(value, new_value) \ - _Py_atomic_store_llong_release(&value, new_value) +#define FT_ATOMIC_STORE_LLONG_RELAXED(value, new_value) \ + _Py_atomic_store_llong_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_LLONG_RELAXED(value) \ _Py_atomic_load_llong_relaxed(&value) -#define FT_ATOMIC_STORE_ULLONG_RELEASE(value, new_value) \ - _Py_atomic_store_ullong_release(&value, new_value) +#define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) \ + _Py_atomic_store_ullong_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) \ _Py_atomic_load_ullong_relaxed(&value) diff --git a/Python/structmember.c b/Python/structmember.c index 5fbde435031631..28b1a2597e3f66 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -191,7 +191,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - FT_ATOMIC_STORE_UCHAR_RELEASE(*(unsigned char*)addr, (unsigned char)long_val); + FT_ATOMIC_STORE_UCHAR_RELAXED(*(unsigned char*)addr, (unsigned char)long_val); if ((long_val > UCHAR_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned char"); break; @@ -200,7 +200,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - FT_ATOMIC_STORE_SHORT_RELEASE(*(short*)addr, (short)long_val); + FT_ATOMIC_STORE_SHORT_RELAXED(*(short*)addr, (short)long_val); if ((long_val > SHRT_MAX) || (long_val < SHRT_MIN)) WARN("Truncation of value to short"); break; @@ -209,7 +209,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - FT_ATOMIC_STORE_USHORT_RELEASE(*(unsigned short*)addr, (unsigned short)long_val); + FT_ATOMIC_STORE_USHORT_RELAXED(*(unsigned short*)addr, (unsigned short)long_val); if ((long_val > USHRT_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned short"); break; @@ -218,7 +218,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - FT_ATOMIC_STORE_INT_RELEASE(*(int *)addr, (int)long_val); + FT_ATOMIC_STORE_INT_RELAXED(*(int *)addr, (int)long_val); if ((long_val > INT_MAX) || (long_val < INT_MIN)) WARN("Truncation of value to int"); break; @@ -236,7 +236,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - FT_ATOMIC_STORE_UINT_RELEASE(*(unsigned int *)addr, (unsigned int)(unsigned long)long_val); + FT_ATOMIC_STORE_UINT_RELAXED(*(unsigned int *)addr, (unsigned int)(unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -245,7 +245,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - FT_ATOMIC_STORE_UINT_RELEASE(*(unsigned int *)addr, (unsigned int)ulong_val); + FT_ATOMIC_STORE_UINT_RELAXED(*(unsigned int *)addr, (unsigned int)ulong_val); if (ulong_val > UINT_MAX) { WARN("Truncation of value to unsigned int"); } @@ -253,7 +253,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case Py_T_LONG:{ - FT_ATOMIC_STORE_LONG_RELEASE(*(long*)addr, PyLong_AsLong(v)); + FT_ATOMIC_STORE_LONG_RELAXED(*(long*)addr, PyLong_AsLong(v)); if ((*(long*)addr == -1) && PyErr_Occurred()) return -1; break; @@ -271,7 +271,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - FT_ATOMIC_STORE_ULONG(*(unsigned long *)addr, (unsigned long)long_val); + FT_ATOMIC_STORE_ULONG_RELAXED(*(unsigned long *)addr, (unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -280,12 +280,12 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - FT_ATOMIC_STORE_ULONG(*(unsigned long *)addr, ulong_val); + FT_ATOMIC_STORE_ULONG_RELAXED(*(unsigned long *)addr, ulong_val); } break; } case Py_T_PYSSIZET:{ - FT_ATOMIC_STORE_SSIZE_RELEASE(*(Py_ssize_t*)addr, PyLong_AsSsize_t(v)); + FT_ATOMIC_STORE_SSIZE_RELAXED(*(Py_ssize_t*)addr, PyLong_AsSsize_t(v)); if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; @@ -295,11 +295,11 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) double double_val = PyFloat_AsDouble(v); if ((double_val == -1) && PyErr_Occurred()) return -1; - FT_ATOMIC_STORE_FLOAT_RELEASE(*(float*)addr, (float)double_val); + FT_ATOMIC_STORE_FLOAT_RELAXED(*(float*)addr, (float)double_val); break; } case Py_T_DOUBLE: - FT_ATOMIC_STORE_DOUBLE_RELEASE(*(double*)addr, PyFloat_AsDouble(v)); + FT_ATOMIC_STORE_DOUBLE_RELAXED(*(double*)addr, PyFloat_AsDouble(v)); if ((*(double*)addr == -1) && PyErr_Occurred()) return -1; break; @@ -329,7 +329,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; case Py_T_LONGLONG:{ long long value = PyLong_AsLongLong(v); - FT_ATOMIC_STORE_LLONG_RELEASE(*(long long*)addr, value); + FT_ATOMIC_STORE_LLONG_RELAXED(*(long long*)addr, value); if ((value == -1) && PyErr_Occurred()) return -1; break; @@ -345,7 +345,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - FT_ATOMIC_STORE_ULLONG_RELEASE(*(unsigned long long *)addr, (unsigned long long)(long long)long_val); + FT_ATOMIC_STORE_ULLONG_RELAXED(*(unsigned long long *)addr, (unsigned long long)(long long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -354,7 +354,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulonglong_val == (unsigned long long)-1 && PyErr_Occurred()) { return -1; } - FT_ATOMIC_STORE_ULLONG_RELEASE(*(unsigned long long *)addr, ulonglong_val); + FT_ATOMIC_STORE_ULLONG_RELAXED(*(unsigned long long *)addr, ulonglong_val); } break; } From b3b07d50280869c998cdea84b5b4c39d9626085e Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 29 Aug 2024 14:28:43 +0200 Subject: [PATCH 28/31] default build --- .../internal/pycore_pyatomic_ft_wrappers.h | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index f7b81f757b33e8..d755d03a5fa190 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -133,29 +133,29 @@ extern "C" { #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value -#define FT_ATOMIC_STORE_CHAR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value -#define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UCHAR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_SHORT_RELAXED(value) value -#define FT_ATOMIC_STORE_SHORT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value -#define FT_ATOMIC_STORE_USHORT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_INT_RELAXED(value) value -#define FT_ATOMIC_STORE_INT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UINT_RELAXED(value) value -#define FT_ATOMIC_STORE_UINT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_LONG_RELAXED(value) value -#define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) value = new_value -#define FT_ATOMIC_STORE_ULONG(value, new_value) value = new_value -#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_LONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_ULONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) value -#define FT_ATOMIC_STORE_FLOAT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) value -#define FT_ATOMIC_STORE_DOUBLE_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_DOUBLE_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_LLONG_RELAXED(value) value -#define FT_ATOMIC_STORE_LLONG_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_LLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value -#define FT_ATOMIC_STORE_ULLONG_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #endif From 761257edb63ece49135aaf29880db85d2dbd0827 Mon Sep 17 00:00:00 2001 From: Daniele Parmeggiani <8658291+dpdani@users.noreply.github.com> Date: Sun, 22 Sep 2024 12:02:52 +0200 Subject: [PATCH 29/31] Apply suggestions from code review Co-authored-by: Sam Gross --- Include/cpython/pyatomic_msc.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index a7600114a64aef..d155955df0cddf 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -956,25 +956,25 @@ _Py_atomic_store_char_relaxed(char *obj, char value) static inline void _Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) { - *(volatile unsigned char*)obj = value; + *(volatile unsigned char *)obj = value; } static inline void _Py_atomic_store_short_relaxed(short *obj, short value) { - *(volatile short*)obj = value; + *(volatile short *)obj = value; } static inline void _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) { - *(volatile unsigned short*)obj = value; + *(volatile unsigned short *)obj = value; } static inline void _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) { - *(volatile unsigned int*)obj = value; + *(volatile unsigned int *)obj = value; } static inline void @@ -998,7 +998,7 @@ _Py_atomic_store_double_relaxed(double *obj, double value) static inline void _Py_atomic_store_llong_relaxed(long long *obj, long long value) { - *(volatile long long*)obj = value; + *(volatile long long *)obj = value; } From d045747320ecc5da41c8f05db8b198e1f1650374 Mon Sep 17 00:00:00 2001 From: dpdani Date: Sun, 22 Sep 2024 12:07:26 +0200 Subject: [PATCH 30/31] missing load --- Python/structmember.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/structmember.c b/Python/structmember.c index 28b1a2597e3f66..7097593e5301b3 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -80,7 +80,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyUnicode_FromString((char*)addr); break; case Py_T_CHAR: { - char value = 0; + char value = FT_ATOMIC_LOAD_CHAR_RELAXED(*addr); v = PyUnicode_FromStringAndSize(&value, 1); break; } From c1b4c358838d50a8f407a37c805a8982a5334453 Mon Sep 17 00:00:00 2001 From: dpdani Date: Sat, 2 Nov 2024 18:19:01 +0100 Subject: [PATCH 31/31] fixes --- Lib/test/test_capi/test_structmembers.py | 13 +++++++++ Modules/_testcapi/structmember.c | 4 +-- Python/structmember.c | 35 +++++++++++++----------- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py index 6b27dc512a7d15..ae9168fc39243f 100644 --- a/Lib/test/test_capi/test_structmembers.py +++ b/Lib/test/test_capi/test_structmembers.py @@ -37,6 +37,9 @@ def _make_test_object(cls): 9.99999,# T_FLOAT 10.1010101010, # T_DOUBLE "hi", # T_STRING_INPLACE + 12, # T_LONGLONG + 13, # T_ULONGLONG + "c", # T_CHAR ) @@ -162,6 +165,16 @@ def test_inplace_string(self): self.assertRaises(TypeError, setattr, ts, "T_STRING_INPLACE", "s") self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") + def test_char(self): + ts = self.ts + self.assertEqual(ts.T_CHAR, "c") + ts.T_CHAR = "z" + self.assertEqual(ts.T_CHAR, "z") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", "") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", b"a") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", bytearray(b"b")) + self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") + class ReadWriteTests_OldAPI(ReadWriteTests, unittest.TestCase): cls = _test_structmembersType_OldAPI diff --git a/Modules/_testcapi/structmember.c b/Modules/_testcapi/structmember.c index f136a25e316d46..c1861db18c4af2 100644 --- a/Modules/_testcapi/structmember.c +++ b/Modules/_testcapi/structmember.c @@ -60,7 +60,7 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "T_FLOAT", "T_DOUBLE", "T_STRING_INPLACE", "T_LONGLONG", "T_ULONGLONG", "T_CHAR", NULL}; - static const char fmt[] = "|bbBhHiIlknfds#LK"; + static const char fmt[] = "|bbBhHiIlknfds#LKC"; test_structmembers *ob; const char *s = NULL; Py_ssize_t string_len = 0; @@ -135,7 +135,7 @@ static struct PyMemberDef test_members[] = { {"T_STRING_INPLACE", T_STRING_INPLACE, offsetof(test_structmembers, structmembers.inplace_member), 0, NULL}, {"T_LONGLONG", T_LONGLONG, offsetof(test_structmembers, structmembers.longlong_member), 0, NULL}, {"T_ULONGLONG", T_ULONGLONG, offsetof(test_structmembers, structmembers.ulonglong_member), 0, NULL}, - {"T_CHAR", Py_T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, + {"T_CHAR", T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, {NULL} }; diff --git a/Python/structmember.c b/Python/structmember.c index 7097593e5301b3..d36e049d6b5d20 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -80,8 +80,8 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyUnicode_FromString((char*)addr); break; case Py_T_CHAR: { - char value = FT_ATOMIC_LOAD_CHAR_RELAXED(*addr); - v = PyUnicode_FromStringAndSize(&value, 1); + char char_val = FT_ATOMIC_LOAD_CHAR_RELAXED(*addr); + v = PyUnicode_FromStringAndSize(&char_val, 1); break; } case _Py_T_OBJECT: @@ -252,12 +252,13 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) } break; } - case Py_T_LONG:{ - FT_ATOMIC_STORE_LONG_RELAXED(*(long*)addr, PyLong_AsLong(v)); - if ((*(long*)addr == -1) && PyErr_Occurred()) + case Py_T_LONG: { + const long long_val = PyLong_AsLong(v); + if ((long_val == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_LONG_RELAXED(*(long*)addr, long_val); break; - } + } case Py_T_ULONG: { /* XXX: For compatibility, accept negative int values as well. */ @@ -284,13 +285,13 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) } break; } - case Py_T_PYSSIZET:{ - FT_ATOMIC_STORE_SSIZE_RELAXED(*(Py_ssize_t*)addr, PyLong_AsSsize_t(v)); - if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) - && PyErr_Occurred()) - return -1; + case Py_T_PYSSIZET: { + const Py_ssize_t ssize_val = PyLong_AsSsize_t(v); + if ((ssize_val == (Py_ssize_t)-1) && PyErr_Occurred()) + return -1; + FT_ATOMIC_STORE_SSIZE_RELAXED(*(Py_ssize_t*)addr, ssize_val); break; - } + } case Py_T_FLOAT:{ double double_val = PyFloat_AsDouble(v); if ((double_val == -1) && PyErr_Occurred()) @@ -298,11 +299,13 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) FT_ATOMIC_STORE_FLOAT_RELAXED(*(float*)addr, (float)double_val); break; } - case Py_T_DOUBLE: - FT_ATOMIC_STORE_DOUBLE_RELAXED(*(double*)addr, PyFloat_AsDouble(v)); - if ((*(double*)addr == -1) && PyErr_Occurred()) + case Py_T_DOUBLE: { + const double double_val = PyFloat_AsDouble(v); + if ((double_val == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_DOUBLE_RELAXED(*(double *) addr, double_val); break; + } case _Py_T_OBJECT: case Py_T_OBJECT_EX: Py_BEGIN_CRITICAL_SECTION(obj); @@ -329,9 +332,9 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; case Py_T_LONGLONG:{ long long value = PyLong_AsLongLong(v); - FT_ATOMIC_STORE_LLONG_RELAXED(*(long long*)addr, value); if ((value == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_LLONG_RELAXED(*(long long*)addr, value); break; } case Py_T_ULONGLONG: {