Index: uspace/lib/c/generic/adt/hash_table.c =================================================================== --- uspace/lib/c/generic/adt/hash_table.c (revision 21799398fb5b1afa28852a26b87fcf58caf15a28) +++ uspace/lib/c/generic/adt/hash_table.c (revision e367939cb9b0dd524632e74c1ac4bd5acb315e5a) @@ -134,5 +134,5 @@ free(h->bucket); - h->bucket = 0; + h->bucket = NULL; h->bucket_cnt = 0; } Index: uspace/lib/c/generic/async.c =================================================================== --- uspace/lib/c/generic/async.c (revision 21799398fb5b1afa28852a26b87fcf58caf15a28) +++ uspace/lib/c/generic/async.c (revision e367939cb9b0dd524632e74c1ac4bd5acb315e5a) @@ -167,5 +167,5 @@ /** Async framework global futex */ -atomic_t async_futex = FUTEX_INITIALIZER; +futex_t async_futex = FUTEX_INITIALIZER; /** Number of threads waiting for IPC in the kernel. */ Index: uspace/lib/c/generic/fibril.c =================================================================== --- uspace/lib/c/generic/fibril.c (revision 21799398fb5b1afa28852a26b87fcf58caf15a28) +++ uspace/lib/c/generic/fibril.c (revision e367939cb9b0dd524632e74c1ac4bd5acb315e5a) @@ -49,4 +49,9 @@ #include #include +#include + +#ifdef FUTEX_UPGRADABLE +#include +#endif /** @@ -54,5 +59,5 @@ * serialized_list and manager_list. */ -static atomic_t fibril_futex = FUTEX_INITIALIZER; +static futex_t fibril_futex = FUTEX_INITIALIZER; static LIST_INITIALIZE(ready_list); @@ -83,4 +88,8 @@ { fibril_t *fibril = __tcb_get()->fibril_data; + +#ifdef FUTEX_UPGRADABLE + rcu_register_fibril(); +#endif /* Call the implementing function. */ @@ -146,5 +155,5 @@ int retval = 0; - futex_down(&fibril_futex); + futex_lock(&fibril_futex); if (stype == FIBRIL_PREEMPT && list_empty(&ready_list)) @@ -168,7 +177,7 @@ if ((stype == FIBRIL_TO_MANAGER) || (stype == FIBRIL_FROM_DEAD)) { while (list_empty(&manager_list)) { - futex_up(&fibril_futex); + futex_unlock(&fibril_futex); async_create_manager(); - futex_down(&fibril_futex); + futex_lock(&fibril_futex); } } @@ -203,5 +212,5 @@ } - return 1; /* futex_up already done here */ + return 1; /* futex_unlock already done here */ } @@ -246,10 +255,17 @@ list_remove(&dstf->link); - futex_up(&fibril_futex); + futex_unlock(&fibril_futex); + +#ifdef FUTEX_UPGRADABLE + if (stype == FIBRIL_FROM_DEAD) { + rcu_deregister_fibril(); + } +#endif + context_restore(&dstf->ctx); /* not reached */ ret_0: - futex_up(&fibril_futex); + futex_unlock(&fibril_futex); return retval; } @@ -318,5 +334,5 @@ fibril_t *fibril = (fibril_t *) fid; - futex_down(&fibril_futex); + futex_lock(&fibril_futex); if ((fibril->flags & FIBRIL_SERIALIZED)) @@ -325,5 +341,5 @@ list_append(&fibril->link, &ready_list); - futex_up(&fibril_futex); + futex_unlock(&fibril_futex); } @@ -338,7 +354,7 @@ fibril_t *fibril = (fibril_t *) fid; - futex_down(&fibril_futex); + futex_lock(&fibril_futex); list_append(&fibril->link, &manager_list); - futex_up(&fibril_futex); + futex_unlock(&fibril_futex); } @@ -346,10 +362,10 @@ void fibril_remove_manager(void) { - futex_down(&fibril_futex); + futex_lock(&fibril_futex); if (!list_empty(&manager_list)) list_remove(list_first(&manager_list)); - futex_up(&fibril_futex); + futex_unlock(&fibril_futex); } Index: uspace/lib/c/generic/futex.c =================================================================== --- uspace/lib/c/generic/futex.c (revision 21799398fb5b1afa28852a26b87fcf58caf15a28) +++ uspace/lib/c/generic/futex.c (revision e367939cb9b0dd524632e74c1ac4bd5acb315e5a) @@ -35,7 +35,4 @@ #include #include -#include -#include -#include /** Initialize futex counter. @@ -47,63 +44,26 @@ void futex_initialize(futex_t *futex, int val) { - atomic_set(futex, val); + atomic_set(&futex->val, val); } -/** Try to down the futex. - * - * @param futex Futex. - * - * @return Non-zero if the futex was acquired. - * @return Zero if the futex was not acquired. - * - */ -int futex_trydown(futex_t *futex) + +#ifdef FUTEX_UPGRADABLE + +int _upgrade_futexes = 0; +static futex_t upg_and_wait_futex = FUTEX_INITIALIZER; + +void futex_upgrade_all_and_wait(void) { - int rc; - - rc = cas(futex, 1, 0); - CS_ENTER_BARRIER(); - - return rc; + futex_down(&upg_and_wait_futex); + + if (!_upgrade_futexes) { + rcu_assign(_upgrade_futexes, 1); + _rcu_synchronize(BM_BLOCK_THREAD); + } + + futex_up(&upg_and_wait_futex); } -/** Down the futex. - * - * @param futex Futex. - * - * @return ENOENT if there is no such virtual address. - * @return Zero in the uncontended case. - * @return Otherwise one of ESYNCH_OK_ATOMIC or ESYNCH_OK_BLOCKED. - * - */ -int futex_down(futex_t *futex) -{ - atomic_signed_t nv; - - nv = (atomic_signed_t) atomic_predec(futex); - CS_ENTER_BARRIER(); - if (nv < 0) - return __SYSCALL1(SYS_FUTEX_SLEEP, (sysarg_t) &futex->count); - - return 0; -} - -/** Up the futex. - * - * @param futex Futex. - * - * @return ENOENT if there is no such virtual address. - * @return Zero in the uncontended case. - * - */ -int futex_up(futex_t *futex) -{ - CS_LEAVE_BARRIER(); - - if ((atomic_signed_t) atomic_postinc(futex) < 0) - return __SYSCALL1(SYS_FUTEX_WAKEUP, (sysarg_t) &futex->count); - - return 0; -} +#endif /** @} Index: uspace/lib/c/generic/ipc.c =================================================================== --- uspace/lib/c/generic/ipc.c (revision 21799398fb5b1afa28852a26b87fcf58caf15a28) +++ uspace/lib/c/generic/ipc.c (revision e367939cb9b0dd524632e74c1ac4bd5acb315e5a) @@ -81,5 +81,5 @@ LIST_INITIALIZE(queued_calls); -static atomic_t ipc_futex = FUTEX_INITIALIZER; +static futex_t ipc_futex = FUTEX_INITIALIZER; /** Send asynchronous message via syscall. @@ -136,10 +136,10 @@ if (!call) { /* Nothing to do regardless if failed or not */ - futex_up(&ipc_futex); + futex_unlock(&ipc_futex); return; } if (callid == (ipc_callid_t) IPC_CALLRET_FATAL) { - futex_up(&ipc_futex); + futex_unlock(&ipc_futex); /* Call asynchronous handler with error code */ @@ -152,5 +152,5 @@ if (callid == (ipc_callid_t) IPC_CALLRET_TEMPORARY) { - futex_up(&ipc_futex); + futex_unlock(&ipc_futex); call->u.msg.phoneid = phoneid; @@ -175,5 +175,5 @@ /* Add call to the list of dispatched calls */ list_append(&call->list, &dispatched_calls); - futex_up(&ipc_futex); + futex_unlock(&ipc_futex); } @@ -219,5 +219,5 @@ */ - futex_down(&ipc_futex); + futex_lock(&ipc_futex); ipc_callid_t callid = __SYSCALL6(SYS_IPC_CALL_ASYNC_FAST, phoneid, imethod, arg1, arg2, arg3, arg4); @@ -226,6 +226,8 @@ if (!call) { call = ipc_prepare_async(private, callback); - if (!call) + if (!call) { + futex_unlock(&ipc_futex); return; + } } @@ -289,5 +291,5 @@ */ - futex_down(&ipc_futex); + futex_lock(&ipc_futex); ipc_callid_t callid = ipc_call_async_internal(phoneid, &call->u.msg.data); @@ -384,7 +386,7 @@ call->u.callid = callid; - futex_down(&ipc_futex); + futex_lock(&ipc_futex); list_append(&call->list, &dispatched_calls); - futex_up(&ipc_futex); + futex_unlock(&ipc_futex); } @@ -412,5 +414,5 @@ callid &= ~IPC_CALLID_ANSWERED; - futex_down(&ipc_futex); + futex_lock(&ipc_futex); link_t *item; @@ -423,5 +425,5 @@ list_remove(&call->list); - futex_up(&ipc_futex); + futex_unlock(&ipc_futex); if (call->callback) @@ -434,5 +436,5 @@ } - futex_up(&ipc_futex); + futex_unlock(&ipc_futex); } Index: uspace/lib/c/generic/libc.c =================================================================== --- uspace/lib/c/generic/libc.c (revision 21799398fb5b1afa28852a26b87fcf58caf15a28) +++ uspace/lib/c/generic/libc.c (revision e367939cb9b0dd524632e74c1ac4bd5acb315e5a) @@ -52,7 +52,12 @@ #include "private/io.h" +#ifdef FUTEX_UPGRADABLE +#include +#endif + #ifdef CONFIG_RTLD #include #endif + static bool env_setup = false; @@ -62,5 +67,4 @@ /* Initialize user task run-time environment */ __malloc_init(); - __async_init(); fibril_t *fibril = fibril_setup(); @@ -72,4 +76,10 @@ /* Save the PCB pointer */ __pcb = (pcb_t *) pcb_ptr; + +#ifdef FUTEX_UPGRADABLE + rcu_register_fibril(); +#endif + + __async_init(); /* The basic run-time environment is setup */ Index: uspace/lib/c/generic/malloc.c =================================================================== --- uspace/lib/c/generic/malloc.c (revision 21799398fb5b1afa28852a26b87fcf58caf15a28) +++ uspace/lib/c/generic/malloc.c (revision e367939cb9b0dd524632e74c1ac4bd5acb315e5a) @@ -200,5 +200,5 @@ do { \ if (!(expr)) {\ - futex_up(&malloc_futex); \ + heap_unlock(); \ assert_abort(#expr, __FILE__, __LINE__); \ } \ @@ -210,4 +210,69 @@ #endif /* NDEBUG */ + + +#ifdef FUTEX_UPGRADABLE +/** True if the heap may be accessed from multiple threads. */ +static bool multithreaded = false; + +/** Makes accesses to the heap thread safe. */ +void malloc_enable_multithreaded(void) +{ + multithreaded = true; +} + +/** Serializes access to the heap from multiple threads. */ +static inline void heap_lock(void) +{ + if (multithreaded) { + futex_down(&malloc_futex); + } else { + /* + * Malloc never switches fibrils while the heap is locked. + * Similarly, it never creates new threads from within the + * locked region. Therefore, if there are no other threads + * except this one, the whole operation will complete without + * any interruptions. + */ + } +} + +/** Serializes access to the heap from multiple threads. */ +static inline void heap_unlock(void) +{ + if (multithreaded) { + futex_up(&malloc_futex); + } else { + /* + * Malloc never switches fibrils while the heap is locked. + * Similarly, it never creates new threads from within the + * locked region. Therefore, if there are no other threads + * except this one, the whole operation will complete without + * any interruptions. + */ + } +} + +#else + +/** Makes accesses to the heap thread safe. */ +void malloc_enable_multithreaded(void) +{ + /* No-op. Already using thread-safe heap locking operations. */ +} + +/** Serializes access to the heap from multiple threads. */ +static inline void heap_lock(void) +{ + futex_down(&malloc_futex); +} + +/** Serializes access to the heap from multiple threads. */ +static inline void heap_unlock(void) +{ + futex_up(&malloc_futex); +} +#endif + /** Initialize a heap block @@ -785,8 +850,8 @@ void *malloc(const size_t size) { - futex_down(&malloc_futex); + heap_lock(); void *block = malloc_internal(size, BASE_ALIGN); - futex_up(&malloc_futex); - + heap_unlock(); + return block; } @@ -807,9 +872,9 @@ size_t palign = 1 << (fnzb(max(sizeof(void *), align) - 1) + 1); - - futex_down(&malloc_futex); + + heap_lock(); void *block = malloc_internal(size, palign); - futex_up(&malloc_futex); - + heap_unlock(); + return block; } @@ -828,5 +893,5 @@ return malloc(size); - futex_down(&malloc_futex); + heap_lock(); /* Calculate the position of the header. */ @@ -885,5 +950,5 @@ } - futex_up(&malloc_futex); + heap_unlock(); if (reloc) { @@ -908,5 +973,5 @@ return; - futex_down(&malloc_futex); + heap_lock(); /* Calculate the position of the header. */ @@ -953,13 +1018,13 @@ heap_shrink(area); - futex_up(&malloc_futex); + heap_unlock(); } void *heap_check(void) { - futex_down(&malloc_futex); + heap_lock(); if (first_heap_area == NULL) { - futex_up(&malloc_futex); + heap_unlock(); return (void *) -1; } @@ -975,5 +1040,5 @@ (((uintptr_t) area->start % PAGE_SIZE) != 0) || (((uintptr_t) area->end % PAGE_SIZE) != 0)) { - futex_up(&malloc_futex); + heap_unlock(); return (void *) area; } @@ -986,5 +1051,5 @@ /* Check heap block consistency */ if (head->magic != HEAP_BLOCK_HEAD_MAGIC) { - futex_up(&malloc_futex); + heap_unlock(); return (void *) head; } @@ -994,5 +1059,5 @@ if ((foot->magic != HEAP_BLOCK_FOOT_MAGIC) || (head->size != foot->size)) { - futex_up(&malloc_futex); + heap_unlock(); return (void *) foot; } @@ -1000,5 +1065,5 @@ } - futex_up(&malloc_futex); + heap_unlock(); return NULL; Index: uspace/lib/c/generic/smp_memory_barrier.c =================================================================== --- uspace/lib/c/generic/smp_memory_barrier.c (revision e367939cb9b0dd524632e74c1ac4bd5acb315e5a) +++ uspace/lib/c/generic/smp_memory_barrier.c (revision e367939cb9b0dd524632e74c1ac4bd5acb315e5a) @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup libc + * @{ + */ +/** @file + */ + +#include +#include + +void smp_memory_barrier(void) +{ + __SYSCALL0(SYS_SMP_MEMORY_BARRIER); +} + +/** @} + */ Index: uspace/lib/c/generic/thread.c =================================================================== --- uspace/lib/c/generic/thread.c (revision 21799398fb5b1afa28852a26b87fcf58caf15a28) +++ uspace/lib/c/generic/thread.c (revision e367939cb9b0dd524632e74c1ac4bd5acb315e5a) @@ -46,4 +46,9 @@ #include "private/thread.h" +#ifdef FUTEX_UPGRADABLE +#include +#endif + + /** Main thread function. * @@ -63,4 +68,9 @@ __tcb_set(fibril->tcb); +#ifdef FUTEX_UPGRADABLE + rcu_register_fibril(); + futex_upgrade_all_and_wait(); +#endif + uarg->uspace_thread_function(uarg->uspace_thread_arg); /* @@ -73,4 +83,9 @@ /* If there is a manager, destroy it */ async_destroy_manager(); + +#ifdef FUTEX_UPGRADABLE + rcu_deregister_fibril(); +#endif + fibril_teardown(fibril); @@ -106,4 +121,7 @@ return ENOMEM; } + + /* Make heap thread safe. */ + malloc_enable_multithreaded(); uarg->uspace_entry = (void *) FADDR(__thread_entry);