Index: uspace/lib/c/arch/arm32/src/atomic.c =================================================================== --- uspace/lib/c/arch/arm32/src/atomic.c (revision 4621d2311994bf63dea425ed923239d4ca1babc9) +++ uspace/lib/c/arch/arm32/src/atomic.c (revision 78de83de52a9115dc77b09bb7029403dad8c2fb0) @@ -81,4 +81,38 @@ } +unsigned short __atomic_fetch_add_2(volatile unsigned short *mem, unsigned short val, int model) +{ + (void) model; + + unsigned short ret; + + /* + * The following instructions between labels 1 and 2 constitute a + * Restartable Atomic Seqeunce. Should the sequence be non-atomic, + * the kernel will restart it. + */ + asm volatile ( + "1:\n" + " adr %[ret], 1b\n" + " str %[ret], %[rp0]\n" + " adr %[ret], 2f\n" + " str %[ret], %[rp1]\n" + " ldrh %[ret], %[addr]\n" + " add %[ret], %[ret], %[imm]\n" + " strh %[ret], %[addr]\n" + "2:\n" + : [ret] "=&r" (ret), + [rp0] "=m" (ras_page[0]), + [rp1] "=m" (ras_page[1]), + [addr] "+m" (*mem) + : [imm] "r" (val) + ); + + ras_page[0] = 0; + ras_page[1] = 0xffffffff; + + return ret - val; +} + unsigned __atomic_fetch_add_4(volatile unsigned *mem, unsigned val, int model) { Index: uspace/lib/c/generic/private/futex.h =================================================================== --- uspace/lib/c/generic/private/futex.h (revision 4621d2311994bf63dea425ed923239d4ca1babc9) +++ uspace/lib/c/generic/private/futex.h (revision 78de83de52a9115dc77b09bb7029403dad8c2fb0) @@ -37,5 +37,5 @@ #include -#include +#include #include #include @@ -43,5 +43,5 @@ typedef struct futex { - atomic_t val; + volatile atomic_int val; #ifdef CONFIG_DEBUG_FUTEX void *owner; @@ -53,5 +53,5 @@ #ifdef CONFIG_DEBUG_FUTEX -#define FUTEX_INITIALIZE(val) {{ (val) }, NULL } +#define FUTEX_INITIALIZE(val) { (val) , NULL } #define FUTEX_INITIALIZER FUTEX_INITIALIZE(1) @@ -73,5 +73,5 @@ #else -#define FUTEX_INITIALIZE(val) {{ (val) }} +#define FUTEX_INITIALIZE(val) { (val) } #define FUTEX_INITIALIZER FUTEX_INITIALIZE(1) @@ -107,6 +107,8 @@ // TODO: Add tests for this. - if ((atomic_signed_t) atomic_predec(&futex->val) >= 0) + if (atomic_fetch_sub_explicit(&futex->val, 1, memory_order_acquire) > 0) return EOK; + + /* There wasn't any token. We must defer to the underlying semaphore. */ usec_t timeout; @@ -129,5 +131,5 @@ } - return __SYSCALL2(SYS_FUTEX_SLEEP, (sysarg_t) &futex->val.count, (sysarg_t) timeout); + return __SYSCALL2(SYS_FUTEX_SLEEP, (sysarg_t) futex, (sysarg_t) timeout); } @@ -143,6 +145,6 @@ static inline errno_t futex_up(futex_t *futex) { - if ((atomic_signed_t) atomic_postinc(&futex->val) < 0) - return __SYSCALL1(SYS_FUTEX_WAKEUP, (sysarg_t) &futex->val.count); + if (atomic_fetch_add_explicit(&futex->val, 1, memory_order_release) < 0) + return __SYSCALL1(SYS_FUTEX_WAKEUP, (sysarg_t) futex); return EOK; @@ -152,38 +154,4 @@ const struct timespec *expires) { - if (expires && expires->tv_sec == 0 && expires->tv_nsec == 0) { - /* Nonblocking down. */ - - /* - * Try good old CAS a few times. - * Not too much though, we don't want to bloat the caller. - */ - for (int i = 0; i < 2; i++) { - atomic_signed_t old = atomic_get(&futex->val); - if (old <= 0) - return ETIMEOUT; - - if (cas(&futex->val, old, old - 1)) - return EOK; - } - - // TODO: builtin atomics with relaxed ordering can make this - // faster. - - /* - * If we don't succeed with CAS, we can't just return failure - * because that would lead to spurious failures where - * futex_down_timeout returns ETIMEOUT despite there being - * available tokens. That could break some algorithms. - * We also don't want to loop on CAS indefinitely, because - * that would make the semaphore not wait-free, even when all - * atomic operations and the underlying base semaphore are all - * wait-free. - * Instead, we fall back to regular down_timeout(), with - * an already expired deadline. That way we delegate all these - * concerns to the base semaphore. - */ - } - /* * This combination of a "composable" sleep followed by futex_up() on @@ -208,6 +176,15 @@ { /* - * down_timeout with an already expired deadline should behave like - * trydown. + * We can't just use CAS here. + * If we don't succeed with CAS, we can't return failure + * because that would lead to spurious failures where + * futex_down_timeout returns ETIMEOUT despite there being + * available tokens. That would break some algorithms. + * We also don't want to loop on CAS indefinitely, because + * that would make the semaphore not wait-free, even when all + * atomic operations and the underlying base semaphore are all + * wait-free. + * It's much less trouble (and code bloat) to just do regular + * down_timeout(), with an already expired deadline. */ struct timespec tv = { .tv_sec = 0, .tv_nsec = 0 }; Index: uspace/lib/c/generic/thread/fibril.c =================================================================== --- uspace/lib/c/generic/thread/fibril.c (revision 4621d2311994bf63dea425ed923239d4ca1babc9) +++ uspace/lib/c/generic/thread/fibril.c (revision 78de83de52a9115dc77b09bb7029403dad8c2fb0) @@ -88,5 +88,5 @@ /* This futex serializes access to global data. */ static futex_t fibril_futex = FUTEX_INITIALIZER; -static futex_t ready_semaphore = FUTEX_INITIALIZE(0); +static futex_t ready_semaphore; static long ready_st_count; @@ -117,19 +117,4 @@ } -static inline long _ready_count(void) -{ - /* - * The number of available tokens is always equal to the number - * of fibrils in the ready list + the number of free IPC buffer - * buckets. - */ - - if (multithreaded) - return atomic_get(&ready_semaphore.val); - - _ready_debug_check(); - return ready_st_count; -} - static inline void _ready_up(void) { @@ -152,5 +137,5 @@ } -static atomic_t threads_in_ipc_wait = { 0 }; +static atomic_int threads_in_ipc_wait; /** Function that spans the whole life-cycle of a fibril. @@ -303,5 +288,6 @@ fibril_t *f = list_pop(&ready_list, fibril_t, link); if (!f) - atomic_inc(&threads_in_ipc_wait); + atomic_fetch_add_explicit(&threads_in_ipc_wait, 1, + memory_order_relaxed); if (!locked) futex_unlock(&fibril_futex); @@ -317,5 +303,6 @@ rc = _ipc_wait(&call, expires); - atomic_dec(&threads_in_ipc_wait); + atomic_fetch_sub_explicit(&threads_in_ipc_wait, 1, + memory_order_relaxed); if (rc != EOK && rc != ENOENT) { @@ -386,5 +373,5 @@ _ready_up(); - if (atomic_get(&threads_in_ipc_wait)) { + if (atomic_load_explicit(&threads_in_ipc_wait, memory_order_relaxed)) { DPRINTF("Poking.\n"); /* Wakeup one thread sleeping in SYS_IPC_WAIT. */ @@ -811,5 +798,5 @@ if (!multithreaded) { _ready_debug_check(); - atomic_set(&ready_semaphore.val, ready_st_count); + futex_initialize(&ready_semaphore, ready_st_count); multithreaded = true; } Index: uspace/lib/c/generic/thread/futex.c =================================================================== --- uspace/lib/c/generic/thread/futex.c (revision 4621d2311994bf63dea425ed923239d4ca1babc9) +++ uspace/lib/c/generic/thread/futex.c (revision 78de83de52a9115dc77b09bb7029403dad8c2fb0) @@ -34,5 +34,5 @@ #include -#include +#include #include #include @@ -52,5 +52,5 @@ void futex_initialize(futex_t *futex, int val) { - atomic_set(&futex->val, val); + atomic_store_explicit(&futex->val, val, memory_order_relaxed); } @@ -59,5 +59,5 @@ void __futex_assert_is_locked(futex_t *futex, const char *name) { - void *owner = __atomic_load_n(&futex->owner, __ATOMIC_RELAXED); + void *owner = atomic_load_explicit(&futex->owner, memory_order_relaxed); fibril_t *self = (fibril_t *) fibril_get_id(); if (owner != self) { @@ -69,5 +69,5 @@ void __futex_assert_is_not_locked(futex_t *futex, const char *name) { - void *owner = __atomic_load_n(&futex->owner, __ATOMIC_RELAXED); + void *owner = atomic_load_explicit(&futex->owner, memory_order_relaxed); fibril_t *self = (fibril_t *) fibril_get_id(); if (owner == self) { @@ -91,7 +91,8 @@ futex_down(futex); - void *prev_owner = __atomic_load_n(&futex->owner, __ATOMIC_RELAXED); + void *prev_owner = atomic_load_explicit(&futex->owner, + memory_order_relaxed); assert(prev_owner == NULL); - __atomic_store_n(&futex->owner, self, __ATOMIC_RELAXED); + atomic_store_explicit(&futex->owner, self, memory_order_relaxed); } @@ -101,5 +102,5 @@ DPRINTF("Unlocking futex %s (%p) by fibril %p.\n", name, futex, self); __futex_assert_is_locked(futex, name); - __atomic_store_n(&futex->owner, NULL, __ATOMIC_RELAXED); + atomic_store_explicit(&futex->owner, NULL, memory_order_relaxed); futex_up(futex); } @@ -110,8 +111,9 @@ bool success = futex_trydown(futex); if (success) { - void *owner = __atomic_load_n(&futex->owner, __ATOMIC_RELAXED); + void *owner = atomic_load_explicit(&futex->owner, + memory_order_relaxed); assert(owner == NULL); - __atomic_store_n(&futex->owner, self, __ATOMIC_RELAXED); + atomic_store_explicit(&futex->owner, self, memory_order_relaxed); DPRINTF("Trylock on futex %s (%p) by fibril %p succeeded.\n", name, futex, self); @@ -130,5 +132,5 @@ __futex_assert_is_locked(futex, name); - __atomic_store_n(&futex->owner, new_owner, __ATOMIC_RELAXED); + atomic_store_explicit(&futex->owner, new_owner, memory_order_relaxed); } Index: uspace/lib/c/include/atomic.h =================================================================== --- uspace/lib/c/include/atomic.h (revision 4621d2311994bf63dea425ed923239d4ca1babc9) +++ (revision ) @@ -1,99 +1,0 @@ -/* - * Copyright (c) 2009 Jakub Jermar - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup libc - * @{ - */ -/** @file - */ - -#ifndef LIBC_ATOMIC_H_ -#define LIBC_ATOMIC_H_ - -#include -#include -#include - -typedef size_t atomic_count_t; -typedef ssize_t atomic_signed_t; - -#define PRIua "zu" /**< Format for atomic_count_t. */ - -typedef struct atomic { - volatile atomic_size_t count; -} atomic_t; - -static inline void atomic_set(atomic_t *val, atomic_count_t i) -{ - atomic_store(&val->count, i); -} - -static inline atomic_count_t atomic_get(atomic_t *val) -{ - return atomic_load(&val->count); -} - -static inline bool cas(atomic_t *val, atomic_count_t ov, atomic_count_t nv) -{ - return atomic_compare_exchange_strong(&val->count, &ov, nv); -} - -static inline atomic_count_t atomic_postinc(atomic_t *val) -{ - return atomic_fetch_add(&val->count, 1); -} - -static inline atomic_count_t atomic_postdec(atomic_t *val) -{ - return atomic_fetch_sub(&val->count, 1); -} - -static inline atomic_count_t atomic_preinc(atomic_t *val) -{ - return atomic_postinc(val) + 1; -} - -static inline atomic_count_t atomic_predec(atomic_t *val) -{ - return atomic_postdec(val) - 1; -} - -static inline void atomic_inc(atomic_t *val) -{ - atomic_postinc(val); -} - -static inline void atomic_dec(atomic_t *val) -{ - atomic_postdec(val); -} - -#endif - -/** @} - */ Index: uspace/lib/c/include/atomicdflt.h =================================================================== --- uspace/lib/c/include/atomicdflt.h (revision 4621d2311994bf63dea425ed923239d4ca1babc9) +++ (revision ) @@ -1,69 +1,0 @@ -/* - * Copyright (c) 2006 Jakub Jermar - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/** @addtogroup libc - * @{ - */ -/** @file - */ - -#ifndef LIBC_ATOMICDFLT_H_ -#define LIBC_ATOMICDFLT_H_ - -#ifndef LIBC_ARCH_ATOMIC_H_ -#error This file cannot be included directly, include atomic.h instead. -#endif - -#include -#include - -typedef struct atomic { - volatile atomic_count_t count; -} atomic_t; - -static inline void atomic_set(atomic_t *val, atomic_count_t i) -{ - val->count = i; -} - -static inline atomic_count_t atomic_get(atomic_t *val) -{ - return val->count; -} - -#ifndef CAS -static inline bool cas(atomic_t *val, atomic_count_t ov, atomic_count_t nv) -{ - return __sync_bool_compare_and_swap(&val->count, ov, nv); -} -#endif - -#endif - -/** @} - */ Index: uspace/lib/c/include/refcount.h =================================================================== --- uspace/lib/c/include/refcount.h (revision 4621d2311994bf63dea425ed923239d4ca1babc9) +++ uspace/lib/c/include/refcount.h (revision 78de83de52a9115dc77b09bb7029403dad8c2fb0) @@ -40,20 +40,16 @@ #define LIBC_REFCOUNT_H_ -// TODO: #include - #include -#include +#include #include /* Wrapped in a structure to prevent direct manipulation. */ typedef struct atomic_refcount { - //volatile atomic_int __cnt; - atomic_t __cnt; + volatile atomic_int __cnt; } atomic_refcount_t; static inline void refcount_init(atomic_refcount_t *rc) { - //atomic_store_explicit(&rc->__cnt, 0, memory_order_relaxed); - atomic_set(&rc->__cnt, 0); + atomic_store_explicit(&rc->__cnt, 0, memory_order_relaxed); } @@ -72,8 +68,6 @@ // still needs to be synchronized independently of the refcount. - //int old = atomic_fetch_add_explicit(&rc->__cnt, 1, - // memory_order_relaxed); - - atomic_signed_t old = atomic_postinc(&rc->__cnt); + int old = atomic_fetch_add_explicit(&rc->__cnt, 1, + memory_order_relaxed); /* old < 0 indicates that the function is used incorrectly. */ @@ -94,8 +88,6 @@ // XXX: The decrementers don't need to synchronize with each other, // but they do need to synchronize with the one doing deallocation. - //int old = atomic_fetch_sub_explicit(&rc->__cnt, 1, - // memory_order_release); - - atomic_signed_t old = atomic_postdec(&rc->__cnt); + int old = atomic_fetch_sub_explicit(&rc->__cnt, 1, + memory_order_release); assert(old >= 0); @@ -104,8 +96,14 @@ // XXX: We are holding the last reference, so we must now // synchronize with all the other decrementers. - //int val = atomic_load_explicit(&rc->__cnt, - // memory_order_acquire); - //assert(val == -1); - return true; + + int val = atomic_load_explicit(&rc->__cnt, + memory_order_acquire); + assert(val == -1); + + /* + * The compiler probably wouldn't optimize the memory barrier + * away, but better safe than sorry. + */ + return val < 0; }