Index: uspace/lib/c/generic/private/futex.h =================================================================== --- uspace/lib/c/generic/private/futex.h (revision ffa73c60dc2eba8ce632c52a0be8bbf4ecd562d9) +++ uspace/lib/c/generic/private/futex.h (revision 508b0df1a393bdc2b6b8e8f8f45be1a5be4a868b) @@ -37,5 +37,5 @@ #include -#include +#include #include #include @@ -43,5 +43,5 @@ typedef struct futex { - atomic_t val; + volatile atomic_int val; #ifdef CONFIG_DEBUG_FUTEX void *owner; @@ -53,5 +53,5 @@ #ifdef CONFIG_DEBUG_FUTEX -#define FUTEX_INITIALIZE(val) {{ (val) }, NULL } +#define FUTEX_INITIALIZE(val) { (val) , NULL } #define FUTEX_INITIALIZER FUTEX_INITIALIZE(1) @@ -73,5 +73,5 @@ #else -#define FUTEX_INITIALIZE(val) {{ (val) }} +#define FUTEX_INITIALIZE(val) { (val) } #define FUTEX_INITIALIZER FUTEX_INITIALIZE(1) @@ -107,6 +107,8 @@ // TODO: Add tests for this. - if ((atomic_signed_t) atomic_predec(&futex->val) >= 0) + if (atomic_fetch_sub_explicit(&futex->val, 1, memory_order_acquire) > 0) return EOK; + + /* There wasn't any token. We must defer to the underlying semaphore. */ usec_t timeout; @@ -129,5 +131,5 @@ } - return __SYSCALL2(SYS_FUTEX_SLEEP, (sysarg_t) &futex->val.count, (sysarg_t) timeout); + return __SYSCALL2(SYS_FUTEX_SLEEP, (sysarg_t) futex, (sysarg_t) timeout); } @@ -143,6 +145,6 @@ static inline errno_t futex_up(futex_t *futex) { - if ((atomic_signed_t) atomic_postinc(&futex->val) < 0) - return __SYSCALL1(SYS_FUTEX_WAKEUP, (sysarg_t) &futex->val.count); + if (atomic_fetch_add_explicit(&futex->val, 1, memory_order_release) < 0) + return __SYSCALL1(SYS_FUTEX_WAKEUP, (sysarg_t) futex); return EOK; @@ -152,38 +154,4 @@ const struct timespec *expires) { - if (expires && expires->tv_sec == 0 && expires->tv_nsec == 0) { - /* Nonblocking down. */ - - /* - * Try good old CAS a few times. - * Not too much though, we don't want to bloat the caller. - */ - for (int i = 0; i < 2; i++) { - atomic_signed_t old = atomic_get(&futex->val); - if (old <= 0) - return ETIMEOUT; - - if (cas(&futex->val, old, old - 1)) - return EOK; - } - - // TODO: builtin atomics with relaxed ordering can make this - // faster. - - /* - * If we don't succeed with CAS, we can't just return failure - * because that would lead to spurious failures where - * futex_down_timeout returns ETIMEOUT despite there being - * available tokens. That could break some algorithms. - * We also don't want to loop on CAS indefinitely, because - * that would make the semaphore not wait-free, even when all - * atomic operations and the underlying base semaphore are all - * wait-free. - * Instead, we fall back to regular down_timeout(), with - * an already expired deadline. That way we delegate all these - * concerns to the base semaphore. - */ - } - /* * This combination of a "composable" sleep followed by futex_up() on @@ -208,6 +176,15 @@ { /* - * down_timeout with an already expired deadline should behave like - * trydown. + * We can't just use CAS here. + * If we don't succeed with CAS, we can't return failure + * because that would lead to spurious failures where + * futex_down_timeout returns ETIMEOUT despite there being + * available tokens. That would break some algorithms. + * We also don't want to loop on CAS indefinitely, because + * that would make the semaphore not wait-free, even when all + * atomic operations and the underlying base semaphore are all + * wait-free. + * It's much less trouble (and code bloat) to just do regular + * down_timeout(), with an already expired deadline. */ struct timespec tv = { .tv_sec = 0, .tv_nsec = 0 }; Index: uspace/lib/c/generic/thread/fibril.c =================================================================== --- uspace/lib/c/generic/thread/fibril.c (revision ffa73c60dc2eba8ce632c52a0be8bbf4ecd562d9) +++ uspace/lib/c/generic/thread/fibril.c (revision 508b0df1a393bdc2b6b8e8f8f45be1a5be4a868b) @@ -88,5 +88,5 @@ /* This futex serializes access to global data. */ static futex_t fibril_futex = FUTEX_INITIALIZER; -static futex_t ready_semaphore = FUTEX_INITIALIZE(0); +static futex_t ready_semaphore; static long ready_st_count; @@ -117,19 +117,4 @@ } -static inline long _ready_count(void) -{ - /* - * The number of available tokens is always equal to the number - * of fibrils in the ready list + the number of free IPC buffer - * buckets. - */ - - if (multithreaded) - return atomic_get(&ready_semaphore.val); - - _ready_debug_check(); - return ready_st_count; -} - static inline void _ready_up(void) { @@ -152,5 +137,5 @@ } -static atomic_t threads_in_ipc_wait = { 0 }; +static atomic_int threads_in_ipc_wait; /** Function that spans the whole life-cycle of a fibril. @@ -303,5 +288,6 @@ fibril_t *f = list_pop(&ready_list, fibril_t, link); if (!f) - atomic_inc(&threads_in_ipc_wait); + atomic_fetch_add_explicit(&threads_in_ipc_wait, 1, + memory_order_relaxed); if (!locked) futex_unlock(&fibril_futex); @@ -317,5 +303,6 @@ rc = _ipc_wait(&call, expires); - atomic_dec(&threads_in_ipc_wait); + atomic_fetch_sub_explicit(&threads_in_ipc_wait, 1, + memory_order_relaxed); if (rc != EOK && rc != ENOENT) { @@ -386,5 +373,5 @@ _ready_up(); - if (atomic_get(&threads_in_ipc_wait)) { + if (atomic_load_explicit(&threads_in_ipc_wait, memory_order_relaxed)) { DPRINTF("Poking.\n"); /* Wakeup one thread sleeping in SYS_IPC_WAIT. */ @@ -811,5 +798,5 @@ if (!multithreaded) { _ready_debug_check(); - atomic_set(&ready_semaphore.val, ready_st_count); + futex_initialize(&ready_semaphore, ready_st_count); multithreaded = true; } Index: uspace/lib/c/generic/thread/futex.c =================================================================== --- uspace/lib/c/generic/thread/futex.c (revision ffa73c60dc2eba8ce632c52a0be8bbf4ecd562d9) +++ uspace/lib/c/generic/thread/futex.c (revision 508b0df1a393bdc2b6b8e8f8f45be1a5be4a868b) @@ -34,5 +34,5 @@ #include -#include +#include #include #include @@ -52,5 +52,5 @@ void futex_initialize(futex_t *futex, int val) { - atomic_set(&futex->val, val); + atomic_store_explicit(&futex->val, val, memory_order_relaxed); } @@ -59,5 +59,5 @@ void __futex_assert_is_locked(futex_t *futex, const char *name) { - void *owner = __atomic_load_n(&futex->owner, __ATOMIC_RELAXED); + void *owner = atomic_load_explicit(&futex->owner, memory_order_relaxed); fibril_t *self = (fibril_t *) fibril_get_id(); if (owner != self) { @@ -69,5 +69,5 @@ void __futex_assert_is_not_locked(futex_t *futex, const char *name) { - void *owner = __atomic_load_n(&futex->owner, __ATOMIC_RELAXED); + void *owner = atomic_load_explicit(&futex->owner, memory_order_relaxed); fibril_t *self = (fibril_t *) fibril_get_id(); if (owner == self) { @@ -91,7 +91,8 @@ futex_down(futex); - void *prev_owner = __atomic_load_n(&futex->owner, __ATOMIC_RELAXED); + void *prev_owner = atomic_load_explicit(&futex->owner, + memory_order_relaxed); assert(prev_owner == NULL); - __atomic_store_n(&futex->owner, self, __ATOMIC_RELAXED); + atomic_store_explicit(&futex->owner, self, memory_order_relaxed); } @@ -101,5 +102,5 @@ DPRINTF("Unlocking futex %s (%p) by fibril %p.\n", name, futex, self); __futex_assert_is_locked(futex, name); - __atomic_store_n(&futex->owner, NULL, __ATOMIC_RELAXED); + atomic_store_explicit(&futex->owner, NULL, memory_order_relaxed); futex_up(futex); } @@ -110,8 +111,9 @@ bool success = futex_trydown(futex); if (success) { - void *owner = __atomic_load_n(&futex->owner, __ATOMIC_RELAXED); + void *owner = atomic_load_explicit(&futex->owner, + memory_order_relaxed); assert(owner == NULL); - __atomic_store_n(&futex->owner, self, __ATOMIC_RELAXED); + atomic_store_explicit(&futex->owner, self, memory_order_relaxed); DPRINTF("Trylock on futex %s (%p) by fibril %p succeeded.\n", name, futex, self); @@ -130,5 +132,5 @@ __futex_assert_is_locked(futex, name); - __atomic_store_n(&futex->owner, new_owner, __ATOMIC_RELAXED); + atomic_store_explicit(&futex->owner, new_owner, memory_order_relaxed); }