From 084ebbd08636480eb8a53aafe60df71e8fa2a166 Mon Sep 17 00:00:00 2001 From: John Sanpe Date: Fri, 9 May 2025 08:39:21 +0800 Subject: [PATCH 1/4] feat barrier: added initial barrier apis Signed-off-by: John Sanpe --- include/bfdev/asm-generic/barrier.h | 39 ++++++++++++++++++++++++ include/bfdev/barrier.h | 46 +++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 include/bfdev/asm-generic/barrier.h create mode 100644 include/bfdev/barrier.h diff --git a/include/bfdev/asm-generic/barrier.h b/include/bfdev/asm-generic/barrier.h new file mode 100644 index 00000000..1403c4c6 --- /dev/null +++ b/include/bfdev/asm-generic/barrier.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * Copyright(c) 2023 John Sanpe + */ + +#ifndef _BFDEV_ASM_GENERIC_BARRIER_H_ +#define _BFDEV_ASM_GENERIC_BARRIER_H_ + +#include + +BFDEV_BEGIN_DECLS + +#ifndef bfdev_arch_mb +# define bfdev_arch_mb() __sync_synchronize() +#endif + +#ifndef bfdev_arch_rmb +# define bfdev_arch_rmb() bfdev_arch_mb() +#endif + +#ifndef bfdev_arch_wmb +# define bfdev_arch_wmb() bfdev_arch_mb() +#endif + +#ifndef bfdev_arch_smp_mb +# define bfdev_arch_smp_mb() bfdev_arch_mb() +#endif + +#ifndef bfdev_arch_smp_rmb +# define bfdev_arch_smp_rmb() bfdev_arch_rmb() +#endif + +#ifndef bfdev_arch_smp_wmb +# define bfdev_arch_smp_wmb() bfdev_arch_wmb() +#endif + +BFDEV_END_DECLS + +#endif /* _BFDEV_ASM_GENERIC_BARRIER_H_ */ diff --git a/include/bfdev/barrier.h b/include/bfdev/barrier.h new file mode 100644 index 00000000..e0ede334 --- /dev/null +++ b/include/bfdev/barrier.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: LGPL-3.0-or-later */ +/* + * Copyright(c) 2025 John Sanpe + */ + +#ifndef _BFDEV_BARRIER_H_ +#define _BFDEV_BARRIER_H_ + +#include +#include + +BFDEV_BEGIN_DECLS + +/* Read and write memory barrier */ +#ifndef bfdev_mb +# define bfdev_mb() bfdev_arch_mb() +#endif + +/* Read memory barrier */ +#ifndef bfdev_rmb +# define bfdev_rmb() bfdev_arch_rmb() +#endif + +/* Write memory barrier */ +#ifndef bfdev_wmb +# define bfdev_wmb() bfdev_arch_wmb() +#endif + +/* SMP read and write memory barrier */ +#ifndef bfdev_smp_mb +# define bfdev_smp_mb() bfdev_arch_smp_mb() +#endif + +/* SMP read memory barrier */ +#ifndef bfdev_smp_rmb +# define bfdev_smp_rmb() bfdev_arch_smp_rmb() +#endif + +/* SMP write memory barrier */ +#ifndef bfdev_smp_wmb +# define bfdev_smp_wmb() bfdev_arch_smp_wmb() +#endif + +BFDEV_END_DECLS + +#endif /* _BFDEV_BARRIER_H_ */ From a5b425593d8e8c35d263fc46da73fae6078bf218 Mon Sep 17 00:00:00 2001 From: John Sanpe Date: Fri, 9 May 2025 08:43:24 +0800 Subject: [PATCH 2/4] fixup fifo: use barrier to fixup smp ooo issue Signed-off-by: John Sanpe --- include/bfdev/fifo.h | 4 ++++ src/fifo.c | 1 + 2 files changed, 5 insertions(+) diff --git a/include/bfdev/fifo.h b/include/bfdev/fifo.h index 55b06147..81df21d3 100644 --- a/include/bfdev/fifo.h +++ b/include/bfdev/fifo.h @@ -10,6 +10,7 @@ #include #include #include +#include BFDEV_BEGIN_DECLS @@ -318,6 +319,7 @@ struct bfdev_fifo { ((typeof(__tmp->data)) __fifo->data) : \ (__tmp->buff)) \ [__fifo->out & __tmp->fifo.mask]; \ + bfdev_smp_wmb(); \ } \ } \ __retval; \ @@ -345,6 +347,7 @@ struct bfdev_fifo { ((typeof(__tmp->data)) __fifo->data) : \ (__tmp->buff)) \ [__fifo->out & __tmp->fifo.mask]; \ + bfdev_smp_wmb(); \ ++__fifo->out; \ } \ } \ @@ -373,6 +376,7 @@ struct bfdev_fifo { (__tmp->buff)) \ [__fifo->in & __tmp->fifo.mask] = \ *(typeof(__tmp->data)) &__tvalue; \ + bfdev_smp_wmb(); \ ++__fifo->in; \ } \ } \ diff --git a/src/fifo.c b/src/fifo.c index d53d1a6e..30ae830d 100644 --- a/src/fifo.c +++ b/src/fifo.c @@ -25,6 +25,7 @@ llen = bfdev_min(len, size - offset); \ bfdev_memcpy(copy1, copy2, llen); \ bfdev_memcpy(fold1, fold2, len - llen); \ + bfdev_smp_wmb(); \ } while (0) static __bfdev_always_inline void From 4b0a9545f53c7f7e144d789688469148f99a7bcc Mon Sep 17 00:00:00 2001 From: John Sanpe Date: Sat, 10 May 2025 17:11:59 +0800 Subject: [PATCH 3/4] feat atomic: added support for memory order Signed-off-by: John Sanpe --- include/bfdev/asm-generic/atomic.h | 325 +++++++++++++++++++++--- include/bfdev/atomic.h | 392 +++++++++++++++++++++++++++++ include/bfdev/compiler.h | 3 + 3 files changed, 684 insertions(+), 36 deletions(-) diff --git a/include/bfdev/asm-generic/atomic.h b/include/bfdev/asm-generic/atomic.h index 37f38875..622fd99d 100644 --- a/include/bfdev/asm-generic/atomic.h +++ b/include/bfdev/asm-generic/atomic.h @@ -12,111 +12,364 @@ BFDEV_BEGIN_DECLS -#ifndef bfdev_arch_atomic_read -# define bfdev_arch_atomic_read bfdev_arch_atomic_read -static __bfdev_always_inline bfdev_atomic_t -bfdev_arch_atomic_read(const bfdev_atomic_t *atomic) -{ - return BFDEV_READ_ONCE(*atomic); -} -#endif - -#ifndef bfdev_arch_atomic_write -# define bfdev_arch_atomic_write bfdev_arch_atomic_write -static __bfdev_always_inline void -bfdev_arch_atomic_write(bfdev_atomic_t *atomic, bfdev_atomic_t value) -{ - BFDEV_WRITE_ONCE(*atomic, value); +#define BFDEV_GENERIC_ATOMIC(name, func, order) \ +static __bfdev_always_inline void \ +bfdev_arch_atomic_##name(bfdev_atomic_t *atomic, bfdev_atomic_t value) \ +{ \ + (void)func(atomic, value, order); \ } -#endif -#define BFDEV_GENERIC_ATOMIC(name, func) \ -static __bfdev_always_inline void \ +#define BFDEV_GENERIC_ATOMIC_FETCH(name, func, order) \ +static __bfdev_always_inline bfdev_atomic_t \ bfdev_arch_atomic_##name(bfdev_atomic_t *atomic, bfdev_atomic_t value) \ { \ - (void)func(atomic, value); \ + return func(atomic, value, order); \ } -#define BFDEV_GENERIC_ATOMIC_FETCH(name, func) \ +#define BFDEV_GENERIC_ATOMIC_READ(name, order) \ static __bfdev_always_inline bfdev_atomic_t \ +bfdev_arch_atomic_##name(const bfdev_atomic_t *atomic) \ +{ \ + bfdev_atomic_t value; \ + __atomic_load(atomic, &value, order); \ + return value; \ +} + +#define BFDEV_GENERIC_ATOMIC_WRITE(name, order) \ +static __bfdev_always_inline void \ bfdev_arch_atomic_##name(bfdev_atomic_t *atomic, bfdev_atomic_t value) \ { \ - return func(atomic, value); \ + __atomic_store(atomic, &value, order); \ } +#ifndef bfdev_arch_atomic_read +# define bfdev_arch_atomic_read bfdev_arch_atomic_read +BFDEV_GENERIC_ATOMIC_READ(read, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_read_acquire +# define bfdev_arch_atomic_read_acquire bfdev_arch_atomic_read_acquire +BFDEV_GENERIC_ATOMIC_READ(read_acquire, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_read_relaxed +# define bfdev_arch_atomic_read_relaxed bfdev_arch_atomic_read_relaxed +BFDEV_GENERIC_ATOMIC_READ(read_relaxed, __ATOMIC_RELAXED) +#endif + +#ifndef bfdev_arch_atomic_write +# define bfdev_arch_atomic_write bfdev_arch_atomic_write +BFDEV_GENERIC_ATOMIC_WRITE(write, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_write_release +# define bfdev_arch_atomic_write_release bfdev_arch_atomic_write_release +BFDEV_GENERIC_ATOMIC_WRITE(write_release, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_write_relaxed +# define bfdev_arch_atomic_write_relaxed bfdev_arch_atomic_write_relaxed +BFDEV_GENERIC_ATOMIC_WRITE(write_relaxed, __ATOMIC_RELAXED) +#endif + #ifndef bfdev_arch_atomic_add # define bfdev_arch_atomic_add bfdev_arch_atomic_add -BFDEV_GENERIC_ATOMIC(add, __sync_fetch_and_add) +BFDEV_GENERIC_ATOMIC(add, __atomic_fetch_add, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_add_acquire +# define bfdev_arch_atomic_add_acquire bfdev_arch_atomic_add_acquire +BFDEV_GENERIC_ATOMIC(add_acquire, __atomic_fetch_add, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_add_release +# define bfdev_arch_atomic_add_release bfdev_arch_atomic_add_release +BFDEV_GENERIC_ATOMIC(add_release, __atomic_fetch_add, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_add_relaxed +# define bfdev_arch_atomic_add_relaxed bfdev_arch_atomic_add_relaxed +BFDEV_GENERIC_ATOMIC(add_relaxed, __atomic_fetch_add, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_sub # define bfdev_arch_atomic_sub bfdev_arch_atomic_sub -BFDEV_GENERIC_ATOMIC(sub, __sync_fetch_and_sub) +BFDEV_GENERIC_ATOMIC(sub, __atomic_fetch_sub, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_sub_acquire +# define bfdev_arch_atomic_sub_acquire bfdev_arch_atomic_sub_acquire +BFDEV_GENERIC_ATOMIC(sub_acquire, __atomic_fetch_sub, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_sub_release +# define bfdev_arch_atomic_sub_release bfdev_arch_atomic_sub_release +BFDEV_GENERIC_ATOMIC(sub_release, __atomic_fetch_sub, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_sub_relaxed +# define bfdev_arch_atomic_sub_relaxed bfdev_arch_atomic_sub_relaxed +BFDEV_GENERIC_ATOMIC(sub_relaxed, __atomic_fetch_sub, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_and # define bfdev_arch_atomic_and bfdev_arch_atomic_and -BFDEV_GENERIC_ATOMIC(and, __sync_fetch_and_and) +BFDEV_GENERIC_ATOMIC(and, __atomic_fetch_and, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_and_acquire +# define bfdev_arch_atomic_and_acquire bfdev_arch_atomic_and_acquire +BFDEV_GENERIC_ATOMIC(and_acquire, __atomic_fetch_and, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_and_release +# define bfdev_arch_atomic_and_release bfdev_arch_atomic_and_release +BFDEV_GENERIC_ATOMIC(and_release, __atomic_fetch_and, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_and_relaxed +# define bfdev_arch_atomic_and_relaxed bfdev_arch_atomic_and_relaxed +BFDEV_GENERIC_ATOMIC(and_relaxed, __atomic_fetch_and, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_or # define bfdev_arch_atomic_or bfdev_arch_atomic_or -BFDEV_GENERIC_ATOMIC(or, __sync_fetch_and_or) +BFDEV_GENERIC_ATOMIC(or, __atomic_fetch_or, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_or_acquire +# define bfdev_arch_atomic_or_acquire bfdev_arch_atomic_or_acquire +BFDEV_GENERIC_ATOMIC(or_acquire, __atomic_fetch_or, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_or_release +# define bfdev_arch_atomic_or_release bfdev_arch_atomic_or_release +BFDEV_GENERIC_ATOMIC(or_release, __atomic_fetch_or, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_or_relaxed +# define bfdev_arch_atomic_or_relaxed bfdev_arch_atomic_or_relaxed +BFDEV_GENERIC_ATOMIC(or_relaxed, __atomic_fetch_or, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_xor # define bfdev_arch_atomic_xor bfdev_arch_atomic_xor -BFDEV_GENERIC_ATOMIC(xor, __sync_fetch_and_xor) +BFDEV_GENERIC_ATOMIC(xor, __atomic_fetch_xor, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_xor_acquire +# define bfdev_arch_atomic_xor_acquire bfdev_arch_atomic_xor_acquire +BFDEV_GENERIC_ATOMIC(xor_acquire, __atomic_fetch_xor, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_xor_release +# define bfdev_arch_atomic_xor_release bfdev_arch_atomic_xor_release +BFDEV_GENERIC_ATOMIC(xor_release, __atomic_fetch_xor, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_xor_relaxed +# define bfdev_arch_atomic_xor_relaxed bfdev_arch_atomic_xor_relaxed +BFDEV_GENERIC_ATOMIC(xor_relaxed, __atomic_fetch_xor, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_fetch_add # define bfdev_arch_atomic_fetch_add bfdev_arch_atomic_fetch_add -BFDEV_GENERIC_ATOMIC_FETCH(fetch_add, __sync_fetch_and_add) +BFDEV_GENERIC_ATOMIC_FETCH(fetch_add, __atomic_fetch_add, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_fetch_add_acquire +# define bfdev_arch_atomic_fetch_add_acquire bfdev_arch_atomic_fetch_add_acquire +BFDEV_GENERIC_ATOMIC_FETCH(fetch_add_acquire, __atomic_fetch_add, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_fetch_add_release +# define bfdev_arch_atomic_fetch_add_release bfdev_arch_atomic_fetch_add_release +BFDEV_GENERIC_ATOMIC_FETCH(fetch_add_release, __atomic_fetch_add, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_fetch_add_relaxed +# define bfdev_arch_atomic_fetch_add_relaxed bfdev_arch_atomic_fetch_add_relaxed +BFDEV_GENERIC_ATOMIC_FETCH(fetch_add_relaxed, __atomic_fetch_add, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_fetch_sub # define bfdev_arch_atomic_fetch_sub bfdev_arch_atomic_fetch_sub -BFDEV_GENERIC_ATOMIC_FETCH(fetch_sub, __sync_fetch_and_sub) +BFDEV_GENERIC_ATOMIC_FETCH(fetch_sub, __atomic_fetch_sub, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_fetch_sub_acquire +# define bfdev_arch_atomic_fetch_sub_acquire bfdev_arch_atomic_fetch_sub_acquire +BFDEV_GENERIC_ATOMIC_FETCH(fetch_sub_acquire, __atomic_fetch_sub, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_fetch_sub_release +# define bfdev_arch_atomic_fetch_sub_release bfdev_arch_atomic_fetch_sub_release +BFDEV_GENERIC_ATOMIC_FETCH(fetch_sub_release, __atomic_fetch_sub, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_fetch_sub_relaxed +# define bfdev_arch_atomic_fetch_sub_relaxed bfdev_arch_atomic_fetch_sub_relaxed +BFDEV_GENERIC_ATOMIC_FETCH(fetch_sub_relaxed, __atomic_fetch_sub, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_fetch_and # define bfdev_arch_atomic_fetch_and bfdev_arch_atomic_fetch_and -BFDEV_GENERIC_ATOMIC_FETCH(fetch_and, __sync_fetch_and_and) +BFDEV_GENERIC_ATOMIC_FETCH(fetch_and, __atomic_fetch_and, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_fetch_and_acquire +# define bfdev_arch_atomic_fetch_and_acquire bfdev_arch_atomic_fetch_and_acquire +BFDEV_GENERIC_ATOMIC_FETCH(fetch_and_acquire, __atomic_fetch_and, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_fetch_and_release +# define bfdev_arch_atomic_fetch_and_release bfdev_arch_atomic_fetch_and_release +BFDEV_GENERIC_ATOMIC_FETCH(fetch_and_release, __atomic_fetch_and, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_fetch_and_relaxed +# define bfdev_arch_atomic_fetch_and_relaxed bfdev_arch_atomic_fetch_and_relaxed +BFDEV_GENERIC_ATOMIC_FETCH(fetch_and_relaxed, __atomic_fetch_and, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_fetch_or # define bfdev_arch_atomic_fetch_or bfdev_arch_atomic_fetch_or -BFDEV_GENERIC_ATOMIC_FETCH(fetch_or, __sync_fetch_and_or) +BFDEV_GENERIC_ATOMIC_FETCH(fetch_or, __atomic_fetch_or, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_fetch_or_acquire +# define bfdev_arch_atomic_fetch_or_acquire bfdev_arch_atomic_fetch_or_acquire +BFDEV_GENERIC_ATOMIC_FETCH(fetch_or_acquire, __atomic_fetch_or, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_fetch_or_release +# define bfdev_arch_atomic_fetch_or_release bfdev_arch_atomic_fetch_or_release +BFDEV_GENERIC_ATOMIC_FETCH(fetch_or_release, __atomic_fetch_or, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_fetch_or_relaxed +# define bfdev_arch_atomic_fetch_or_relaxed bfdev_arch_atomic_fetch_or_relaxed +BFDEV_GENERIC_ATOMIC_FETCH(fetch_or_relaxed, __atomic_fetch_or, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_fetch_xor # define bfdev_arch_atomic_fetch_xor bfdev_arch_atomic_fetch_xor -BFDEV_GENERIC_ATOMIC_FETCH(fetch_xor, __sync_fetch_and_xor) +BFDEV_GENERIC_ATOMIC_FETCH(fetch_xor, __atomic_fetch_xor, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_fetch_xor_acquire +# define bfdev_arch_atomic_fetch_xor_acquire bfdev_arch_atomic_fetch_xor_acquire +BFDEV_GENERIC_ATOMIC_FETCH(fetch_xor_acquire, __atomic_fetch_xor, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_fetch_xor_release +# define bfdev_arch_atomic_fetch_xor_release bfdev_arch_atomic_fetch_xor_release +BFDEV_GENERIC_ATOMIC_FETCH(fetch_xor_release, __atomic_fetch_xor, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_fetch_xor_relaxed +# define bfdev_arch_atomic_fetch_xor_relaxed bfdev_arch_atomic_fetch_xor_relaxed +BFDEV_GENERIC_ATOMIC_FETCH(fetch_xor_relaxed, __atomic_fetch_xor, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_add_fetch # define bfdev_arch_atomic_add_fetch bfdev_arch_atomic_add_fetch -BFDEV_GENERIC_ATOMIC_FETCH(add_fetch, __sync_add_and_fetch) +BFDEV_GENERIC_ATOMIC_FETCH(add_fetch, __atomic_add_fetch, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_add_fetch_acquire +# define bfdev_arch_atomic_add_fetch_acquire bfdev_arch_atomic_add_fetch_acquire +BFDEV_GENERIC_ATOMIC_FETCH(add_fetch_acquire, __atomic_add_fetch, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_add_fetch_release +# define bfdev_arch_atomic_add_fetch_release bfdev_arch_atomic_add_fetch_release +BFDEV_GENERIC_ATOMIC_FETCH(add_fetch_release, __atomic_add_fetch, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_add_fetch_relaxed +# define bfdev_arch_atomic_add_fetch_relaxed bfdev_arch_atomic_add_fetch_relaxed +BFDEV_GENERIC_ATOMIC_FETCH(add_fetch_relaxed, __atomic_add_fetch, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_sub_fetch # define bfdev_arch_atomic_sub_fetch bfdev_arch_atomic_sub_fetch -BFDEV_GENERIC_ATOMIC_FETCH(sub_fetch, __sync_sub_and_fetch) +BFDEV_GENERIC_ATOMIC_FETCH(sub_fetch, __atomic_sub_fetch, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_sub_fetch_acquire +# define bfdev_arch_atomic_sub_fetch_acquire bfdev_arch_atomic_sub_fetch_acquire +BFDEV_GENERIC_ATOMIC_FETCH(sub_fetch_acquire, __atomic_sub_fetch, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_sub_fetch_release +# define bfdev_arch_atomic_sub_fetch_release bfdev_arch_atomic_sub_fetch_release +BFDEV_GENERIC_ATOMIC_FETCH(sub_fetch_release, __atomic_sub_fetch, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_sub_fetch_relaxed +# define bfdev_arch_atomic_sub_fetch_relaxed bfdev_arch_atomic_sub_fetch_relaxed +BFDEV_GENERIC_ATOMIC_FETCH(sub_fetch_relaxed, __atomic_sub_fetch, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_and_fetch # define bfdev_arch_atomic_and_fetch bfdev_arch_atomic_and_fetch -BFDEV_GENERIC_ATOMIC_FETCH(and_fetch, __sync_and_and_fetch) +BFDEV_GENERIC_ATOMIC_FETCH(and_fetch, __atomic_and_fetch, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_and_fetch_acquire +# define bfdev_arch_atomic_and_fetch_acquire bfdev_arch_atomic_and_fetch_acquire +BFDEV_GENERIC_ATOMIC_FETCH(and_fetch_acquire, __atomic_and_fetch, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_and_fetch_release +# define bfdev_arch_atomic_and_fetch_release bfdev_arch_atomic_and_fetch_release +BFDEV_GENERIC_ATOMIC_FETCH(and_fetch_release, __atomic_and_fetch, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_and_fetch_relaxed +# define bfdev_arch_atomic_and_fetch_relaxed bfdev_arch_atomic_and_fetch_relaxed +BFDEV_GENERIC_ATOMIC_FETCH(and_fetch_relaxed, __atomic_and_fetch, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_or_fetch # define bfdev_arch_atomic_or_fetch bfdev_arch_atomic_or_fetch -BFDEV_GENERIC_ATOMIC_FETCH(or_fetch, __sync_or_and_fetch) +BFDEV_GENERIC_ATOMIC_FETCH(or_fetch, __atomic_or_fetch, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_or_fetch_acquire +# define bfdev_arch_atomic_or_fetch_acquire bfdev_arch_atomic_or_fetch_acquire +BFDEV_GENERIC_ATOMIC_FETCH(or_fetch_acquire, __atomic_or_fetch, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_or_fetch_release +# define bfdev_arch_atomic_or_fetch_release bfdev_arch_atomic_or_fetch_release +BFDEV_GENERIC_ATOMIC_FETCH(or_fetch_release, __atomic_or_fetch, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_or_fetch_relaxed +# define bfdev_arch_atomic_or_fetch_relaxed bfdev_arch_atomic_or_fetch_relaxed +BFDEV_GENERIC_ATOMIC_FETCH(or_fetch_relaxed, __atomic_or_fetch, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_atomic_xor_fetch # define bfdev_arch_atomic_xor_fetch bfdev_arch_atomic_xor_fetch -BFDEV_GENERIC_ATOMIC_FETCH(xor_fetch, __sync_xor_and_fetch) +BFDEV_GENERIC_ATOMIC_FETCH(xor_fetch, __atomic_xor_fetch, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_atomic_xor_fetch_acquire +# define bfdev_arch_atomic_xor_fetch_acquire bfdev_arch_atomic_xor_fetch_acquire +BFDEV_GENERIC_ATOMIC_FETCH(xor_fetch_acquire, __atomic_xor_fetch, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_atomic_xor_fetch_release +# define bfdev_arch_atomic_xor_fetch_release bfdev_arch_atomic_xor_fetch_release +BFDEV_GENERIC_ATOMIC_FETCH(xor_fetch_release, __atomic_xor_fetch, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_atomic_xor_fetch_relaxed +# define bfdev_arch_atomic_xor_fetch_relaxed bfdev_arch_atomic_xor_fetch_relaxed +BFDEV_GENERIC_ATOMIC_FETCH(xor_fetch_relaxed, __atomic_xor_fetch, __ATOMIC_RELAXED) #endif BFDEV_END_DECLS diff --git a/include/bfdev/atomic.h b/include/bfdev/atomic.h index aba03f00..957c8264 100644 --- a/include/bfdev/atomic.h +++ b/include/bfdev/atomic.h @@ -23,6 +23,22 @@ bfdev_atomic_read(const bfdev_atomic_t *atomic) } #endif +#ifndef bfdev_atomic_read_acquire +static __bfdev_always_inline bfdev_atomic_t +bfdev_atomic_read_acquire(const bfdev_atomic_t *atomic) +{ + return bfdev_arch_atomic_read_acquire(atomic); +} +#endif + +#ifndef bfdev_atomic_read_relaxed +static __bfdev_always_inline bfdev_atomic_t +bfdev_atomic_read_relaxed(const bfdev_atomic_t *atomic) +{ + return bfdev_arch_atomic_read_relaxed(atomic); +} +#endif + /** * bfdev_atomic_write - atomic write variable. * @atomic: pointer of type atomic_t. @@ -36,6 +52,22 @@ bfdev_atomic_write(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_write_release +static __bfdev_always_inline void +bfdev_atomic_write_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_write_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_write_relaxed +static __bfdev_always_inline void +bfdev_atomic_write_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_write_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_add - atomic add variable. * @atomic: pointer of type atomic_t. @@ -49,6 +81,30 @@ bfdev_atomic_add(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_add_acquire +static __bfdev_always_inline void +bfdev_atomic_add_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_add_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_add_release +static __bfdev_always_inline void +bfdev_atomic_add_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_add_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_add_relaxed +static __bfdev_always_inline void +bfdev_atomic_add_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_add_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_sub - atomic subtract variable. * @atomic: pointer of type atomic_t. @@ -62,6 +118,30 @@ bfdev_atomic_sub(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_sub_acquire +static __bfdev_always_inline void +bfdev_atomic_sub_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_sub_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_sub_release +static __bfdev_always_inline void +bfdev_atomic_sub_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_sub_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_sub_relaxed +static __bfdev_always_inline void +bfdev_atomic_sub_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_sub_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_and - atomic and variable. * @atomic: pointer of type atomic_t. @@ -75,6 +155,30 @@ bfdev_atomic_and(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_and_acquire +static __bfdev_always_inline void +bfdev_atomic_and_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_and_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_and_release +static __bfdev_always_inline void +bfdev_atomic_and_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_and_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_and_relaxed +static __bfdev_always_inline void +bfdev_atomic_and_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_and_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_or - atomic or variable. * @atomic: pointer of type atomic_t. @@ -88,6 +192,30 @@ bfdev_atomic_or(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_or_acquire +static __bfdev_always_inline void +bfdev_atomic_or_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_or_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_or_release +static __bfdev_always_inline void +bfdev_atomic_or_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_or_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_or_relaxed +static __bfdev_always_inline void +bfdev_atomic_or_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_or_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_xor - atomic xor variable. * @atomic: pointer of type atomic_t. @@ -101,6 +229,30 @@ bfdev_atomic_xor(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_xor_acquire +static __bfdev_always_inline void +bfdev_atomic_xor_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_xor_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_xor_release +static __bfdev_always_inline void +bfdev_atomic_xor_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_xor_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_xor_relaxed +static __bfdev_always_inline void +bfdev_atomic_xor_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_xor_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_fetch_add - fetch and atomic add variable. * @atomic: pointer of type atomic_t. @@ -114,6 +266,30 @@ bfdev_atomic_fetch_add(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_fetch_add_acquire +static __bfdev_always_inline void +bfdev_atomic_fetch_add_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_add_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_fetch_add_release +static __bfdev_always_inline void +bfdev_atomic_fetch_add_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_add_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_fetch_add_relaxed +static __bfdev_always_inline void +bfdev_atomic_fetch_add_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_add_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_fetch_sub - fetch and atomic subtract variable. * @atomic: pointer of type atomic_t. @@ -127,6 +303,30 @@ bfdev_atomic_fetch_sub(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_fetch_sub_acquire +static __bfdev_always_inline void +bfdev_atomic_fetch_sub_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_sub_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_fetch_sub_release +static __bfdev_always_inline void +bfdev_atomic_fetch_sub_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_sub_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_fetch_sub_relaxed +static __bfdev_always_inline void +bfdev_atomic_fetch_sub_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_sub_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_fetch_and - fetch and atomic and variable. * @atomic: pointer of type atomic_t. @@ -140,6 +340,30 @@ bfdev_atomic_fetch_and(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_fetch_and_acquire +static __bfdev_always_inline void +bfdev_atomic_fetch_and_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_and_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_fetch_and_release +static __bfdev_always_inline void +bfdev_atomic_fetch_and_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_and_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_fetch_and_relaxed +static __bfdev_always_inline void +bfdev_atomic_fetch_and_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_and_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_fetch_or - fetch and atomic or variable. * @atomic: pointer of type atomic_t. @@ -153,6 +377,30 @@ bfdev_atomic_fetch_or(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_fetch_or_acquire +static __bfdev_always_inline void +bfdev_atomic_fetch_or_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_or_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_fetch_or_release +static __bfdev_always_inline void +bfdev_atomic_fetch_or_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_or_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_fetch_or_relaxed +static __bfdev_always_inline void +bfdev_atomic_fetch_or_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_or_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_fetch_xor - fetch and atomic xor variable. * @atomic: pointer of type atomic_t. @@ -166,6 +414,30 @@ bfdev_atomic_fetch_xor(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_fetch_xor_acquire +static __bfdev_always_inline void +bfdev_atomic_fetch_xor_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_xor_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_fetch_xor_release +static __bfdev_always_inline void +bfdev_atomic_fetch_xor_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_xor_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_fetch_xor_relaxed +static __bfdev_always_inline void +bfdev_atomic_fetch_xor_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_fetch_xor_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_add_fetch - atomic add variable and fetch. * @atomic: pointer of type atomic_t. @@ -179,6 +451,30 @@ bfdev_atomic_add_fetch(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_add_fetch_acquire +static __bfdev_always_inline void +bfdev_atomic_add_fetch_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_add_fetch_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_add_fetch_release +static __bfdev_always_inline void +bfdev_atomic_add_fetch_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_add_fetch_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_add_fetch_relaxed +static __bfdev_always_inline void +bfdev_atomic_add_fetch_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_add_fetch_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_sub_fetch - atomic subtract variable and fetch. * @atomic: pointer of type atomic_t. @@ -192,6 +488,30 @@ bfdev_atomic_sub_fetch(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_sub_fetch_acquire +static __bfdev_always_inline void +bfdev_atomic_sub_fetch_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_sub_fetch_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_sub_fetch_release +static __bfdev_always_inline void +bfdev_atomic_sub_fetch_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_sub_fetch_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_sub_fetch_relaxed +static __bfdev_always_inline void +bfdev_atomic_sub_fetch_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_sub_fetch_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_and_fetch - atomic and variable and fetch. * @atomic: pointer of type atomic_t. @@ -205,6 +525,30 @@ bfdev_atomic_and_fetch(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_and_fetch_acquire +static __bfdev_always_inline void +bfdev_atomic_and_fetch_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_and_fetch_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_and_fetch_release +static __bfdev_always_inline void +bfdev_atomic_and_fetch_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_and_fetch_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_and_fetch_relaxed +static __bfdev_always_inline void +bfdev_atomic_and_fetch_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_and_fetch_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_or_fetch - atomic or variable and fetch. * @atomic: pointer of type atomic_t. @@ -218,6 +562,30 @@ bfdev_atomic_or_fetch(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_or_fetch_acquire +static __bfdev_always_inline void +bfdev_atomic_or_fetch_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_or_fetch_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_or_fetch_release +static __bfdev_always_inline void +bfdev_atomic_or_fetch_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_or_fetch_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_or_fetch_relaxed +static __bfdev_always_inline void +bfdev_atomic_or_fetch_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_or_fetch_relaxed(atomic, value); +} +#endif + /** * bfdev_atomic_xor_fetch - atomic xor variable and fetch. * @atomic: pointer of type atomic_t. @@ -231,6 +599,30 @@ bfdev_atomic_xor_fetch(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_atomic_xor_fetch_acquire +static __bfdev_always_inline void +bfdev_atomic_xor_fetch_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_xor_fetch_acquire(atomic, value); +} +#endif + +#ifndef bfdev_atomic_xor_fetch_release +static __bfdev_always_inline void +bfdev_atomic_xor_fetch_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_xor_fetch_release(atomic, value); +} +#endif + +#ifndef bfdev_atomic_xor_fetch_relaxed +static __bfdev_always_inline void +bfdev_atomic_xor_fetch_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + bfdev_arch_atomic_xor_fetch_relaxed(atomic, value); +} +#endif + BFDEV_END_DECLS #endif /* _BFDEV_ATOMIC_H_ */ diff --git a/include/bfdev/compiler.h b/include/bfdev/compiler.h index e227aa51..7ba025dd 100644 --- a/include/bfdev/compiler.h +++ b/include/bfdev/compiler.h @@ -28,6 +28,9 @@ BFDEV_BEGIN_DECLS #ifndef bfdev_barrier # define bfdev_barrier() __bfdev_barrier(:) +#endif + +#ifndef bfdev_barrier_data # define bfdev_barrier_data(ptr) __bfdev_barrier(:"r"(ptr)) #endif From 0d67e35bc3bff2645c9f4b46672e9ad4076706e7 Mon Sep 17 00:00:00 2001 From: John Sanpe Date: Sat, 10 May 2025 17:16:35 +0800 Subject: [PATCH 4/4] feat cmpxchg: added support for memory order Signed-off-by: John Sanpe --- include/bfdev/asm-generic/cmpxchg.h | 107 ++++++++++++++++++++-------- include/bfdev/cmpxchg.h | 84 +++++++++++++++++++++- 2 files changed, 158 insertions(+), 33 deletions(-) diff --git a/include/bfdev/asm-generic/cmpxchg.h b/include/bfdev/asm-generic/cmpxchg.h index 0e169753..ec65072f 100644 --- a/include/bfdev/asm-generic/cmpxchg.h +++ b/include/bfdev/asm-generic/cmpxchg.h @@ -13,47 +13,92 @@ BFDEV_BEGIN_DECLS -#ifndef bfdev_arch_cmpxchg -# define bfdev_arch_cmpxchg bfdev_arch_cmpxchg -static __bfdev_always_inline bfdev_atomic_t -bfdev_arch_cmpxchg(bfdev_atomic_t *atomic, bfdev_atomic_t old, - bfdev_atomic_t value) -{ - return __sync_val_compare_and_swap(atomic, old, value); +#define BFDEV_GENERIC_XCHG(name, order) \ +static __bfdev_always_inline bfdev_atomic_t \ +bfdev_arch_##name(bfdev_atomic_t *atomic, bfdev_atomic_t value) \ +{ \ + bfdev_atomic_t result; \ + __atomic_exchange(atomic, &value, &result, order); \ + return result; \ +} + +#define BFDEV_GENERIC_CMPXCHG(name, weak, okay, fail) \ +static __bfdev_always_inline bfdev_atomic_t \ +bfdev_arch_##name(bfdev_atomic_t *atomic, bfdev_atomic_t old, \ + bfdev_atomic_t value) \ +{ \ + __atomic_compare_exchange(atomic, &old, &value, \ + weak, okay, fail); \ + return old; \ +} + +#define BFDEV_GENERIC_TRY_CMPXCHG(name, weak, okay, fail) \ +static __bfdev_always_inline bfdev_bool \ +bfdev_arch_##name(bfdev_atomic_t *atomic, bfdev_atomic_t *old, \ + bfdev_atomic_t value) \ +{ \ + return __atomic_compare_exchange(atomic, old, &value, \ + weak, okay, fail); \ } -#endif #ifndef bfdev_arch_xchg # define bfdev_arch_xchg bfdev_arch_xchg -static __bfdev_always_inline bfdev_atomic_t -bfdev_arch_xchg(bfdev_atomic_t *atomic, bfdev_atomic_t value) -{ - bfdev_atomic_t prev, result; +BFDEV_GENERIC_XCHG(xchg, __ATOMIC_SEQ_CST) +#endif - do { - prev = *atomic; - result = bfdev_arch_cmpxchg(atomic, prev, value); - } while (bfdev_unlikely(result != prev)); +#ifndef bfdev_arch_xchg_acquire +# define bfdev_arch_xchg_acquire bfdev_arch_xchg_acquire +BFDEV_GENERIC_XCHG(xchg_acquire, __ATOMIC_ACQUIRE) +#endif - return result; -} +#ifndef bfdev_arch_xchg_release +# define bfdev_arch_xchg_release bfdev_arch_xchg_release +BFDEV_GENERIC_XCHG(xchg_release, __ATOMIC_RELEASE) +#endif + +#ifndef bfdev_arch_xchg_relaxed +# define bfdev_arch_xchg_relaxed bfdev_arch_xchg_relaxed +BFDEV_GENERIC_XCHG(xchg_relaxed, __ATOMIC_RELAXED) +#endif + +#ifndef bfdev_arch_cmpxchg +# define bfdev_arch_cmpxchg bfdev_arch_cmpxchg +BFDEV_GENERIC_CMPXCHG(cmpxchg, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_cmpxchg_acquire +# define bfdev_arch_cmpxchg_acquire bfdev_arch_cmpxchg_acquire +BFDEV_GENERIC_CMPXCHG(cmpxchg_acquire, 0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_cmpxchg_release +# define bfdev_arch_cmpxchg_release bfdev_arch_cmpxchg_release +BFDEV_GENERIC_CMPXCHG(cmpxchg_release, 0, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_cmpxchg_relaxed +# define bfdev_arch_cmpxchg_relaxed bfdev_arch_cmpxchg_relaxed +BFDEV_GENERIC_CMPXCHG(cmpxchg_relaxed, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED) #endif #ifndef bfdev_arch_try_cmpxchg # define bfdev_arch_try_cmpxchg bfdev_arch_try_cmpxchg -static __bfdev_always_inline bfdev_bool -bfdev_arch_try_cmpxchg(bfdev_atomic_t *atomic, bfdev_atomic_t *old, - bfdev_atomic_t value) -{ - bfdev_atomic_t result, prev; - - prev = *old; - result = bfdev_arch_cmpxchg(atomic, prev, value); - if (bfdev_unlikely(result != prev)) - *old = result; - - return bfdev_likely(result == prev); -} +BFDEV_GENERIC_TRY_CMPXCHG(try_cmpxchg, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) +#endif + +#ifndef bfdev_arch_try_cmpxchg_acquire +# define bfdev_arch_try_cmpxchg_acquire bfdev_arch_try_cmpxchg_acquire +BFDEV_GENERIC_TRY_CMPXCHG(try_cmpxchg_acquire, 0, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_try_cmpxchg_release +# define bfdev_arch_try_cmpxchg_release bfdev_arch_try_cmpxchg_release +BFDEV_GENERIC_TRY_CMPXCHG(try_cmpxchg_release, 0, __ATOMIC_RELEASE, __ATOMIC_ACQUIRE) +#endif + +#ifndef bfdev_arch_try_cmpxchg_relaxed +# define bfdev_arch_try_cmpxchg_relaxed bfdev_arch_try_cmpxchg_relaxed +BFDEV_GENERIC_TRY_CMPXCHG(try_cmpxchg_relaxed, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED) #endif BFDEV_END_DECLS diff --git a/include/bfdev/cmpxchg.h b/include/bfdev/cmpxchg.h index 759d04d2..9c59a4ff 100644 --- a/include/bfdev/cmpxchg.h +++ b/include/bfdev/cmpxchg.h @@ -19,22 +19,102 @@ bfdev_xchg(bfdev_atomic_t *atomic, bfdev_atomic_t value) } #endif +#ifndef bfdev_xchg_acquire +static __bfdev_always_inline bfdev_atomic_t +bfdev_xchg_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + return bfdev_arch_xchg_acquire(atomic, value); +} +#endif + +#ifndef bfdev_xchg_release +static __bfdev_always_inline bfdev_atomic_t +bfdev_xchg_release(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + return bfdev_arch_xchg_release(atomic, value); +} +#endif + +#ifndef bfdev_xchg_relaxed +static __bfdev_always_inline bfdev_atomic_t +bfdev_xchg_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t value) +{ + return bfdev_arch_xchg_relaxed(atomic, value); +} +#endif + #ifndef bfdev_cmpxchg static __bfdev_always_inline bfdev_atomic_t -bfdev_cmpxchg(bfdev_atomic_t *atomic, bfdev_atomic_t old, bfdev_atomic_t value) +bfdev_cmpxchg(bfdev_atomic_t *atomic, bfdev_atomic_t old, + bfdev_atomic_t value) { return bfdev_arch_cmpxchg(atomic, old, value); } #endif +#ifndef bfdev_cmpxchg_acquire +static __bfdev_always_inline bfdev_atomic_t +bfdev_cmpxchg_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t old, + bfdev_atomic_t value) +{ + return bfdev_arch_cmpxchg_acquire(atomic, old, value); +} +#endif + +#ifndef bfdev_cmpxchg_release +static __bfdev_always_inline bfdev_atomic_t +bfdev_cmpxchg_release(bfdev_atomic_t *atomic, bfdev_atomic_t old, + bfdev_atomic_t value) +{ + return bfdev_arch_cmpxchg_release(atomic, old, value); +} +#endif + +#ifndef bfdev_cmpxchg_relaxed +static __bfdev_always_inline bfdev_atomic_t +bfdev_cmpxchg_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t old, + bfdev_atomic_t value) +{ + return bfdev_arch_cmpxchg_relaxed(atomic, old, value); +} +#endif + #ifndef bfdev_try_cmpxchg static __bfdev_always_inline bfdev_bool -bfdev_try_cmpxchg(bfdev_atomic_t *atomic, bfdev_atomic_t *old, bfdev_atomic_t value) +bfdev_try_cmpxchg(bfdev_atomic_t *atomic, bfdev_atomic_t *old, + bfdev_atomic_t value) { return bfdev_arch_try_cmpxchg(atomic, old, value); } #endif +#ifndef bfdev_try_cmpxchg_acquire +static __bfdev_always_inline bfdev_bool +bfdev_try_cmpxchg_acquire(bfdev_atomic_t *atomic, bfdev_atomic_t *old, + bfdev_atomic_t value) +{ + return bfdev_arch_try_cmpxchg_acquire(atomic, old, value); +} +#endif + +#ifndef bfdev_try_cmpxchg_release +static __bfdev_always_inline bfdev_bool +bfdev_try_cmpxchg_release(bfdev_atomic_t *atomic, bfdev_atomic_t *old, + bfdev_atomic_t value) +{ + return bfdev_arch_try_cmpxchg_release(atomic, old, value); +} +#endif + +#ifndef bfdev_try_cmpxchg_relaxed +static __bfdev_always_inline bfdev_bool +bfdev_try_cmpxchg_relaxed(bfdev_atomic_t *atomic, bfdev_atomic_t *old, + bfdev_atomic_t value) +{ + return bfdev_arch_try_cmpxchg_relaxed(atomic, old, value); +} +#endif + BFDEV_END_DECLS #endif /* _BFDEV_CMPXCHG_H_ */