Index: uspace/lib/c/generic/async/client.c =================================================================== --- uspace/lib/c/generic/async/client.c (revision 7137f74c74c9c2eea4980d18b86a6d9667725946) +++ uspace/lib/c/generic/async/client.c (revision 514d56134040fa72282251a1c8434e3190f9d1b6) @@ -104,5 +104,4 @@ #include #include -#include #include #include @@ -128,5 +127,5 @@ /** Message data */ typedef struct { - awaiter_t wdata; + fibril_event_t received; /** If reply was received. */ @@ -136,7 +135,4 @@ bool forget; - /** If already destroyed. */ - bool destroyed; - /** Pointer to where the answer data is stored. */ ipc_call_t *dataptr; @@ -145,50 +141,11 @@ } amsg_t; -static void to_event_initialize(to_event_t *to) -{ - struct timeval tv = { 0, 0 }; - - to->inlist = false; - to->occurred = false; - link_initialize(&to->link); - to->expires = tv; -} - -static void wu_event_initialize(wu_event_t *wu) -{ - wu->inlist = false; - link_initialize(&wu->link); -} - -void awaiter_initialize(awaiter_t *aw) -{ - aw->fid = 0; - aw->active = false; - to_event_initialize(&aw->to_event); - wu_event_initialize(&aw->wu_event); -} - static amsg_t *amsg_create(void) { - amsg_t *msg = malloc(sizeof(amsg_t)); - if (msg) { - msg->done = false; - msg->forget = false; - msg->destroyed = false; - msg->dataptr = NULL; - msg->retval = EINVAL; - awaiter_initialize(&msg->wdata); - } - - return msg; + return calloc(1, sizeof(amsg_t)); } static void amsg_destroy(amsg_t *msg) { - if (!msg) - return; - - assert(!msg->destroyed); - msg->destroyed = true; free(msg); } @@ -251,22 +208,14 @@ msg->retval = IPC_GET_RETVAL(*data); - /* Copy data after futex_down, just in case the call was detached */ + /* Copy data inside lock, just in case the call was detached */ if ((msg->dataptr) && (data)) *msg->dataptr = *data; - write_barrier(); - - /* Remove message from timeout list */ - if (msg->wdata.to_event.inlist) - list_remove(&msg->wdata.to_event.link); - msg->done = true; if (msg->forget) { - assert(msg->wdata.active); amsg_destroy(msg); - } else if (!msg->wdata.active) { - msg->wdata.active = true; - fibril_add_ready(msg->wdata.fid); + } else { + fibril_notify(&msg->received); } @@ -301,5 +250,4 @@ msg->dataptr = dataptr; - msg->wdata.active = true; errno_t rc = ipc_call_async_4(exch->phone, imethod, arg1, arg2, arg3, @@ -343,5 +291,4 @@ msg->dataptr = dataptr; - msg->wdata.active = true; errno_t rc = ipc_call_async_5(exch->phone, imethod, arg1, arg2, arg3, @@ -371,24 +318,6 @@ amsg_t *msg = (amsg_t *) amsgid; - - futex_lock(&async_futex); - - assert(!msg->forget); - assert(!msg->destroyed); - - if (msg->done) { - futex_unlock(&async_futex); - goto done; - } - - msg->wdata.fid = fibril_get_id(); - msg->wdata.active = false; - msg->wdata.to_event.inlist = false; - - /* Leave the async_futex locked when entering this function */ - fibril_switch(FIBRIL_FROM_BLOCKED); - futex_unlock(&async_futex); - -done: + fibril_wait_for(&msg->received); + if (retval) *retval = msg->retval; @@ -420,14 +349,4 @@ amsg_t *msg = (amsg_t *) amsgid; - - futex_lock(&async_futex); - - assert(!msg->forget); - assert(!msg->destroyed); - - if (msg->done) { - futex_unlock(&async_futex); - goto done; - } /* @@ -438,36 +357,12 @@ timeout = 0; - getuptime(&msg->wdata.to_event.expires); - tv_add_diff(&msg->wdata.to_event.expires, timeout); - - /* - * Current fibril is inserted as waiting regardless of the - * "size" of the timeout. - * - * Checking for msg->done and immediately bailing out when - * timeout == 0 would mean that the manager fibril would never - * run (consider single threaded program). - * Thus the IPC answer would be never retrieved from the kernel. - * - * Notice that the actual delay would be very small because we - * - switch to manager fibril - * - the manager sees expired timeout - * - and thus adds us back to ready queue - * - manager switches back to some ready fibril - * (prior it, it checks for incoming IPC). - * - */ - msg->wdata.fid = fibril_get_id(); - msg->wdata.active = false; - async_insert_timeout(&msg->wdata); - - /* Leave the async_futex locked when entering this function */ - fibril_switch(FIBRIL_FROM_BLOCKED); - futex_unlock(&async_futex); - - if (!msg->done) - return ETIMEOUT; - -done: + struct timeval expires; + getuptime(&expires); + tv_add_diff(&expires, timeout); + + errno_t rc = fibril_wait_timeout(&msg->received, &expires); + if (rc != EOK) + return rc; + if (retval) *retval = msg->retval; @@ -475,5 +370,5 @@ amsg_destroy(msg); - return 0; + return EOK; } @@ -494,5 +389,4 @@ assert(!msg->forget); - assert(!msg->destroyed); futex_lock(&async_futex); @@ -506,49 +400,4 @@ futex_unlock(&async_futex); -} - -/** Wait for specified time. - * - * The current fibril is suspended but the thread continues to execute. - * - * @param timeout Duration of the wait in microseconds. - * - */ -void fibril_usleep(suseconds_t timeout) -{ - awaiter_t awaiter; - awaiter_initialize(&awaiter); - - awaiter.fid = fibril_get_id(); - - getuptime(&awaiter.to_event.expires); - tv_add_diff(&awaiter.to_event.expires, timeout); - - futex_lock(&async_futex); - - async_insert_timeout(&awaiter); - - /* Leave the async_futex locked when entering this function */ - fibril_switch(FIBRIL_FROM_BLOCKED); - futex_unlock(&async_futex); -} - -/** Delay execution for the specified number of seconds - * - * @param sec Number of seconds to sleep - */ -void fibril_sleep(unsigned int sec) -{ - /* - * Sleep in 1000 second steps to support - * full argument range - */ - - while (sec > 0) { - unsigned int period = (sec > 1000) ? 1000 : sec; - - fibril_usleep(period * 1000000); - sec -= period; - } } @@ -716,5 +565,4 @@ msg->dataptr = &result; - msg->wdata.active = true; errno_t rc = ipc_call_async_4(phone, IPC_M_CONNECT_ME_TO, Index: uspace/lib/c/generic/async/server.c =================================================================== --- uspace/lib/c/generic/async/server.c (revision 7137f74c74c9c2eea4980d18b86a6d9667725946) +++ uspace/lib/c/generic/async/server.c (revision 514d56134040fa72282251a1c8434e3190f9d1b6) @@ -104,5 +104,4 @@ #include #include -#include #include #include @@ -118,4 +117,5 @@ #include #include +#include #include #include @@ -127,7 +127,4 @@ /** Async framework global futex */ futex_t async_futex = FUTEX_INITIALIZER; - -/** Number of threads waiting for IPC in the kernel. */ -static atomic_t threads_in_ipc_wait = { 0 }; /** Call data */ @@ -148,5 +145,6 @@ /* Server connection data */ typedef struct { - awaiter_t wdata; + /** Fibril handling the connection. */ + fid_t fid; /** Hash table link. */ @@ -161,4 +159,7 @@ /** Link to the client tracking structure. */ client_t *client; + + /** Message event. */ + fibril_event_t msg_arrived; /** Messages that should be delivered to this fibril. */ @@ -251,5 +252,4 @@ /* The remaining structures are guarded by async_futex. */ static hash_table_t conn_hash_table; -static LIST_INITIALIZE(timeout_list); static size_t client_key_hash(void *key) @@ -487,9 +487,10 @@ ipc_answer_0(call->cap_handle, ENOMEM); - return (uintptr_t) NULL; + return (fid_t) NULL; } conn->in_task_id = in_task_id; conn->in_phone_hash = in_phone_hash; + conn->msg_arrived = FIBRIL_EVENT_INIT; list_initialize(&conn->msg_queue); conn->close_chandle = CAP_NIL; @@ -503,8 +504,7 @@ /* We will activate the fibril ASAP */ - conn->wdata.active = true; - conn->wdata.fid = fibril_create(connection_fibril, conn); - - if (conn->wdata.fid == 0) { + conn->fid = fibril_create(connection_fibril, conn); + + if (conn->fid == 0) { free(conn); @@ -512,5 +512,5 @@ ipc_answer_0(call->cap_handle, ENOMEM); - return (uintptr_t) NULL; + return (fid_t) NULL; } @@ -521,7 +521,7 @@ futex_unlock(&async_futex); - fibril_add_ready(conn->wdata.fid); - - return conn->wdata.fid; + fibril_add_ready(conn->fid); + + return conn->fid; } @@ -566,5 +566,5 @@ fid_t fid = async_new_connection(answer.in_task_id, phone_hash, NULL, handler, data); - if (fid == (uintptr_t) NULL) + if (fid == (fid_t) NULL) return ENOMEM; @@ -602,30 +602,4 @@ }; -/** Sort in current fibril's timeout request. - * - * @param wd Wait data of the current fibril. - * - */ -void async_insert_timeout(awaiter_t *wd) -{ - assert(wd); - - wd->to_event.occurred = false; - wd->to_event.inlist = true; - - link_t *tmp = timeout_list.head.next; - while (tmp != &timeout_list.head) { - awaiter_t *cur = - list_get_instance(tmp, awaiter_t, to_event.link); - - if (tv_gteq(&cur->to_event.expires, &wd->to_event.expires)) - break; - - tmp = tmp->next; - } - - list_insert_before(&wd->to_event.link, tmp); -} - /** Try to route a call to an appropriate connection fibril. * @@ -657,4 +631,5 @@ connection_t *conn = hash_table_get_inst(link, connection_t, link); + // FIXME: malloc in critical section msg_t *msg = malloc(sizeof(*msg)); if (!msg) { @@ -670,15 +645,5 @@ /* If the connection fibril is waiting for an event, activate it */ - if (!conn->wdata.active) { - - /* If in timeout list, remove it */ - if (conn->wdata.to_event.inlist) { - conn->wdata.to_event.inlist = false; - list_remove(&conn->wdata.to_event.link); - } - - conn->wdata.active = true; - fibril_add_ready(conn->wdata.fid); - } + fibril_notify(&conn->msg_arrived); futex_unlock(&async_futex); @@ -987,11 +952,13 @@ connection_t *conn = fibril_connection; + struct timeval tv; + struct timeval *expires = NULL; + if (usecs) { + getuptime(&tv); + tv_add_diff(&tv, usecs); + expires = &tv; + } + futex_lock(&async_futex); - - if (usecs) { - getuptime(&conn->wdata.to_event.expires); - tv_add_diff(&conn->wdata.to_event.expires, usecs); - } else - conn->wdata.to_event.inlist = false; /* If nothing in queue, wait until something arrives */ @@ -1011,23 +978,12 @@ } - if (usecs) - async_insert_timeout(&conn->wdata); - - conn->wdata.active = false; - - /* - * Note: the current fibril will be rescheduled either due to a - * timeout or due to an arriving message destined to it. In the - * former case, handle_expired_timeouts() and, in the latter - * case, route_call() will perform the wakeup. - */ - fibril_switch(FIBRIL_FROM_BLOCKED); - - if ((usecs) && (conn->wdata.to_event.occurred) && - (list_empty(&conn->msg_queue))) { - /* If we timed out -> exit */ - futex_unlock(&async_futex); + // TODO: replace with cvar + futex_unlock(&async_futex); + + errno_t rc = fibril_wait_timeout(&conn->msg_arrived, expires); + if (rc == ETIMEOUT) return false; - } + + futex_lock(&async_futex); } @@ -1126,54 +1082,4 @@ } -/** Fire all timeouts that expired. */ -static suseconds_t handle_expired_timeouts(unsigned int *flags) -{ - /* Make sure the async_futex is held. */ - futex_assert_is_locked(&async_futex); - - struct timeval tv; - getuptime(&tv); - - bool fired = false; - - link_t *cur = list_first(&timeout_list); - while (cur != NULL) { - awaiter_t *waiter = - list_get_instance(cur, awaiter_t, to_event.link); - - if (tv_gt(&waiter->to_event.expires, &tv)) { - if (fired) { - *flags = SYNCH_FLAGS_NON_BLOCKING; - return 0; - } - *flags = 0; - return tv_sub_diff(&waiter->to_event.expires, &tv); - } - - list_remove(&waiter->to_event.link); - waiter->to_event.inlist = false; - waiter->to_event.occurred = true; - - /* - * Redundant condition? - * The fibril should not be active when it gets here. - */ - if (!waiter->active) { - waiter->active = true; - fibril_add_ready(waiter->fid); - fired = true; - } - - cur = list_first(&timeout_list); - } - - if (fired) { - *flags = SYNCH_FLAGS_NON_BLOCKING; - return 0; - } - - return SYNCH_NO_TIMEOUT; -} - /** Endless loop dispatching incoming calls and answers. * @@ -1183,24 +1089,9 @@ static errno_t async_manager_worker(void) { + ipc_call_t call; + errno_t rc; + while (true) { - futex_lock(&async_futex); - fibril_switch(FIBRIL_FROM_MANAGER); - - /* - * The switch only returns when there is no non-manager fibril - * it can run. - */ - - unsigned int flags = SYNCH_FLAGS_NONE; - suseconds_t next_timeout = handle_expired_timeouts(&flags); - futex_unlock(&async_futex); - - atomic_inc(&threads_in_ipc_wait); - - ipc_call_t call; - errno_t rc = ipc_wait(&call, next_timeout, flags); - - atomic_dec(&threads_in_ipc_wait); - + rc = fibril_ipc_wait(&call, NULL); if (rc == EOK) handle_call(&call); @@ -1225,15 +1116,9 @@ /** Add one manager to manager list. */ -void async_create_manager(void) +fid_t async_create_manager(void) { fid_t fid = fibril_create_generic(async_manager_fibril, NULL, PAGE_SIZE); - if (fid != 0) - fibril_add_manager(fid); -} - -/** Remove one manager from manager list */ -void async_destroy_manager(void) -{ - fibril_remove_manager(); + fibril_start(fid); + return fid; } @@ -1252,4 +1137,6 @@ ¬ification_hash_table_ops)) abort(); + + async_create_manager(); } @@ -1342,11 +1229,4 @@ return EOK; -} - -/** Interrupt one thread of this task from waiting for IPC. */ -void async_poke(void) -{ - if (atomic_get(&threads_in_ipc_wait) > 0) - ipc_poke(); } @@ -1834,6 +1714,6 @@ __noreturn void async_manager(void) { - futex_lock(&async_futex); - fibril_switch(FIBRIL_FROM_DEAD); + fibril_event_t ever = FIBRIL_EVENT_INIT; + fibril_wait_for(&ever); __builtin_unreachable(); } Index: uspace/lib/c/generic/fibril.c =================================================================== --- uspace/lib/c/generic/fibril.c (revision 7137f74c74c9c2eea4980d18b86a6d9667725946) +++ uspace/lib/c/generic/fibril.c (revision 514d56134040fa72282251a1c8434e3190f9d1b6) @@ -2,4 +2,5 @@ * Copyright (c) 2006 Ondrej Palkovsky * Copyright (c) 2007 Jakub Jermar + * Copyright (c) 2018 CZ.NIC, z.s.p.o. * All rights reserved. * @@ -39,26 +40,69 @@ #include #include -#include #include -#include -#include #include #include #include -#include - + +#include +#include +#include +#include #include "private/thread.h" #include "private/fibril.h" #include "private/libc.h" -/** - * This futex serializes access to ready_list, - * manager_list and fibril_list. - */ +#define DPRINTF(...) ((void)0) + +/** Member of timeout_list. */ +typedef struct { + link_t link; + struct timeval expires; + fibril_event_t *event; +} _timeout_t; + +typedef struct { + errno_t rc; + link_t link; + ipc_call_t *call; + fibril_event_t event; +} _ipc_waiter_t; + +typedef struct { + errno_t rc; + link_t link; + ipc_call_t call; +} _ipc_buffer_t; + +typedef enum { + SWITCH_FROM_DEAD, + SWITCH_FROM_HELPER, + SWITCH_FROM_YIELD, + SWITCH_FROM_BLOCKED, +} _switch_type_t; + +static bool multithreaded = false; + +/* This futex serializes access to global data. */ static futex_t fibril_futex = FUTEX_INITIALIZER; +static futex_t ready_semaphore = FUTEX_INITIALIZE(0); static LIST_INITIALIZE(ready_list); -static LIST_INITIALIZE(manager_list); static LIST_INITIALIZE(fibril_list); +static LIST_INITIALIZE(timeout_list); + +static futex_t ipc_lists_futex = FUTEX_INITIALIZER; +static LIST_INITIALIZE(ipc_waiter_list); +static LIST_INITIALIZE(ipc_buffer_list); +static LIST_INITIALIZE(ipc_buffer_free_list); + +/* Only used as unique markers for triggered events. */ +static fibril_t _fibril_event_triggered; +static fibril_t _fibril_event_timed_out; +#define _EVENT_INITIAL (NULL) +#define _EVENT_TRIGGERED (&_fibril_event_triggered) +#define _EVENT_TIMED_OUT (&_fibril_event_timed_out) + +static atomic_t threads_in_ipc_wait = { 0 }; /** Function that spans the whole life-cycle of a fibril. @@ -69,17 +113,14 @@ * */ -static void fibril_main(void) -{ - /* fibril_futex and async_futex are locked when a fibril is started. */ +static void _fibril_main(void) +{ + /* fibril_futex is locked when a fibril is started. */ futex_unlock(&fibril_futex); - futex_unlock(&async_futex); fibril_t *fibril = fibril_self(); /* Call the implementing function. */ - fibril->retval = fibril->func(fibril->arg); - - futex_lock(&async_futex); - fibril_switch(FIBRIL_FROM_DEAD); + fibril_exit(fibril->func(fibril->arg)); + /* Not reached */ } @@ -116,11 +157,9 @@ } -void fibril_teardown(fibril_t *fibril, bool locked) -{ - if (!locked) - futex_lock(&fibril_futex); +void fibril_teardown(fibril_t *fibril) +{ + futex_lock(&fibril_futex); list_remove(&fibril->all_link); - if (!locked) - futex_unlock(&fibril_futex); + futex_unlock(&fibril_futex); if (fibril->is_freeable) { @@ -130,102 +169,342 @@ } -/** Switch from the current fibril. - * - * The async_futex must be held when entering this function, - * and is still held on return. - * - * @param stype Switch type. One of FIBRIL_PREEMPT, FIBRIL_TO_MANAGER, - * FIBRIL_FROM_MANAGER, FIBRIL_FROM_DEAD. The parameter - * describes the circumstances of the switch. - * - * @return 0 if there is no ready fibril, - * @return 1 otherwise. - * - */ -int fibril_switch(fibril_switch_type_t stype) -{ - /* Make sure the async_futex is held. */ - futex_assert_is_locked(&async_futex); +/** + * Event notification with a given reason. + * + * @param reason Reason of the notification. + * Can be either _EVENT_TRIGGERED or _EVENT_TIMED_OUT. + */ +static fibril_t *_fibril_trigger_internal(fibril_event_t *event, fibril_t *reason) +{ + assert(reason != _EVENT_INITIAL); + assert(reason == _EVENT_TIMED_OUT || reason == _EVENT_TRIGGERED); + + futex_assert_is_locked(&fibril_futex); + + if (event->fibril == _EVENT_INITIAL) { + event->fibril = reason; + return NULL; + } + + if (event->fibril == _EVENT_TIMED_OUT) { + assert(reason == _EVENT_TRIGGERED); + event->fibril = reason; + return NULL; + } + + if (event->fibril == _EVENT_TRIGGERED) { + /* Already triggered. Nothing to do. */ + return NULL; + } + + fibril_t *f = event->fibril; + event->fibril = reason; + + assert(f->sleep_event == event); + return f; +} + +static errno_t _ipc_wait(ipc_call_t *call, const struct timeval *expires) +{ + if (!expires) + return ipc_wait(call, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NONE); + + if (expires->tv_sec == 0) + return ipc_wait(call, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NON_BLOCKING); + + struct timeval now; + getuptime(&now); + + if (tv_gteq(&now, expires)) + return ipc_wait(call, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NON_BLOCKING); + + return ipc_wait(call, tv_sub_diff(expires, &now), SYNCH_FLAGS_NONE); +} + +/* + * Waits until a ready fibril is added to the list, or an IPC message arrives. + * Returns NULL on timeout and may also return NULL if returning from IPC + * wait after new ready fibrils are added. + */ +static fibril_t *_ready_list_pop(const struct timeval *expires, bool locked) +{ + if (locked) { + futex_assert_is_locked(&fibril_futex); + assert(expires); + /* Must be nonblocking. */ + assert(expires->tv_sec == 0); + } else { + futex_assert_is_not_locked(&fibril_futex); + } + + if (!multithreaded) { + /* + * The number of available tokens is always equal to the number + * of fibrils in the ready list + the number of free IPC buffer + * buckets. + */ + + assert(atomic_get(&ready_semaphore.val) == + list_count(&ready_list) + list_count(&ipc_buffer_free_list)); + } + + errno_t rc = futex_down_timeout(&ready_semaphore, expires); + + if (rc != EOK) + return NULL; + + /* + * Once we acquire a token from ready_semaphore, there are two options. + * Either there is a ready fibril in the list, or it's our turn to + * call `ipc_wait_cycle()`. There is one extra token on the semaphore + * for each entry of the call buffer. + */ + + + if (!locked) + futex_lock(&fibril_futex); + fibril_t *f = list_pop(&ready_list, fibril_t, link); + if (!f) + atomic_inc(&threads_in_ipc_wait); + if (!locked) + futex_unlock(&fibril_futex); + + if (f) + return f; + + if (!multithreaded) + assert(list_empty(&ipc_buffer_list)); + + /* No fibril is ready, IPC wait it is. */ + ipc_call_t call = { 0 }; + rc = _ipc_wait(&call, expires); + + atomic_dec(&threads_in_ipc_wait); + + if (rc != EOK && rc != ENOENT) { + /* Return token. */ + futex_up(&ready_semaphore); + return NULL; + } + + /* + * We might get ENOENT due to a poke. + * In that case, we propagate the null call out of fibril_ipc_wait(), + * because poke must result in that call returning. + */ + + /* + * If a fibril is already waiting for IPC, we wake up the fibril, + * and return the token to ready_semaphore. + * If there is no fibril waiting, we pop a buffer bucket and + * put our call there. The token then returns when the bucket is + * returned. + */ + + if (!locked) + futex_lock(&fibril_futex); + + futex_lock(&ipc_lists_futex); + + + _ipc_waiter_t *w = list_pop(&ipc_waiter_list, _ipc_waiter_t, link); + if (w) { + *w->call = call; + w->rc = rc; + /* We switch to the woken up fibril immediately if possible. */ + f = _fibril_trigger_internal(&w->event, _EVENT_TRIGGERED); + + /* Return token. */ + futex_up(&ready_semaphore); + } else { + _ipc_buffer_t *buf = list_pop(&ipc_buffer_free_list, _ipc_buffer_t, link); + assert(buf); + *buf = (_ipc_buffer_t) { .call = call, .rc = rc }; + list_append(&buf->link, &ipc_buffer_list); + } + + futex_unlock(&ipc_lists_futex); + + if (!locked) + futex_unlock(&fibril_futex); + + return f; +} + +static fibril_t *_ready_list_pop_nonblocking(bool locked) +{ + struct timeval tv = { .tv_sec = 0, .tv_usec = 0 }; + return _ready_list_pop(&tv, locked); +} + +static void _ready_list_push(fibril_t *f) +{ + if (!f) + return; + + futex_assert_is_locked(&fibril_futex); + + /* Enqueue in ready_list. */ + list_append(&f->link, &ready_list); + futex_up(&ready_semaphore); + + if (atomic_get(&threads_in_ipc_wait)) { + DPRINTF("Poking.\n"); + /* Wakeup one thread sleeping in SYS_IPC_WAIT. */ + ipc_poke(); + } +} + +/* Blocks the current fibril until an IPC call arrives. */ +static errno_t _wait_ipc(ipc_call_t *call, const struct timeval *expires) +{ + futex_assert_is_not_locked(&fibril_futex); + + futex_lock(&ipc_lists_futex); + _ipc_buffer_t *buf = list_pop(&ipc_buffer_list, _ipc_buffer_t, link); + if (buf) { + *call = buf->call; + errno_t rc = buf->rc; + + /* Return to freelist. */ + list_append(&buf->link, &ipc_buffer_free_list); + /* Return IPC wait token. */ + futex_up(&ready_semaphore); + + futex_unlock(&ipc_lists_futex); + return rc; + } + + _ipc_waiter_t w = { .call = call }; + list_append(&w.link, &ipc_waiter_list); + futex_unlock(&ipc_lists_futex); + + errno_t rc = fibril_wait_timeout(&w.event, expires); + if (rc == EOK) + return w.rc; + + futex_lock(&ipc_lists_futex); + if (link_in_use(&w.link)) + list_remove(&w.link); + else + rc = w.rc; + futex_unlock(&ipc_lists_futex); + return rc; +} + +/** Fire all timeouts that expired. */ +static struct timeval *_handle_expired_timeouts(struct timeval *next_timeout) +{ + struct timeval tv; + getuptime(&tv); futex_lock(&fibril_futex); + while (!list_empty(&timeout_list)) { + link_t *cur = list_first(&timeout_list); + _timeout_t *to = list_get_instance(cur, _timeout_t, link); + + if (tv_gt(&to->expires, &tv)) { + *next_timeout = to->expires; + futex_unlock(&fibril_futex); + return next_timeout; + } + + list_remove(&to->link); + + _ready_list_push(_fibril_trigger_internal( + to->event, _EVENT_TIMED_OUT)); + } + + futex_unlock(&fibril_futex); + return NULL; +} + +/** + * Clean up after a dead fibril from which we restored context, if any. + * Called after a switch is made and fibril_futex is unlocked. + */ +static void _fibril_cleanup_dead(void) +{ fibril_t *srcf = fibril_self(); - fibril_t *dstf = NULL; - - /* Choose a new fibril to run */ - if (list_empty(&ready_list)) { - if (stype == FIBRIL_PREEMPT || stype == FIBRIL_FROM_MANAGER) { - // FIXME: This means that as long as there is a fibril - // that only yields, IPC messages are never retrieved. - futex_unlock(&fibril_futex); - return 0; - } - - /* If we are going to manager and none exists, create it */ - while (list_empty(&manager_list)) { - futex_unlock(&fibril_futex); - async_create_manager(); - futex_lock(&fibril_futex); - } - - dstf = list_get_instance(list_first(&manager_list), - fibril_t, link); - } else { - dstf = list_get_instance(list_first(&ready_list), fibril_t, - link); - } - - list_remove(&dstf->link); - if (stype == FIBRIL_FROM_DEAD) + if (!srcf->clean_after_me) + return; + + void *stack = srcf->clean_after_me->stack; + assert(stack); + as_area_destroy(stack); + fibril_teardown(srcf->clean_after_me); + srcf->clean_after_me = NULL; +} + +/** Switch to a fibril. */ +static void _fibril_switch_to(_switch_type_t type, fibril_t *dstf, bool locked) +{ + if (!locked) + futex_lock(&fibril_futex); + else + futex_assert_is_locked(&fibril_futex); + + fibril_t *srcf = fibril_self(); + assert(srcf); + assert(dstf); + + switch (type) { + case SWITCH_FROM_YIELD: + _ready_list_push(srcf); + break; + case SWITCH_FROM_DEAD: dstf->clean_after_me = srcf; - - /* Put the current fibril into the correct run list */ - switch (stype) { - case FIBRIL_PREEMPT: - list_append(&srcf->link, &ready_list); break; - case FIBRIL_FROM_MANAGER: - list_append(&srcf->link, &manager_list); + case SWITCH_FROM_HELPER: + case SWITCH_FROM_BLOCKED: break; - case FIBRIL_FROM_DEAD: - case FIBRIL_FROM_BLOCKED: - // Nothing. - break; - } - - /* Bookkeeping. */ + } + + dstf->thread_ctx = srcf->thread_ctx; + srcf->thread_ctx = NULL; + + /* Just some bookkeeping to allow better debugging of futex locks. */ futex_give_to(&fibril_futex, dstf); - futex_give_to(&async_futex, dstf); /* Swap to the next fibril. */ context_swap(&srcf->ctx, &dstf->ctx); - /* Restored by another fibril! */ - - /* Must be after context_swap()! */ - futex_unlock(&fibril_futex); - - if (srcf->clean_after_me) { - /* - * Cleanup after the dead fibril from which we - * restored context here. - */ - void *stack = srcf->clean_after_me->stack; - if (stack) { - /* - * This check is necessary because a - * thread could have exited like a - * normal fibril using the - * FIBRIL_FROM_DEAD switch type. In that - * case, its fibril will not have the - * stack member filled. - */ - as_area_destroy(stack); + assert(srcf == fibril_self()); + assert(srcf->thread_ctx); + + if (!locked) { + /* Must be after context_swap()! */ + futex_unlock(&fibril_futex); + _fibril_cleanup_dead(); + } +} + +/** + * Main function for a helper fibril. + * The helper fibril executes on threads in the lightweight fibril pool when + * there is no fibril ready to run. Its only purpose is to block until + * another fibril is ready, or a timeout expires, or an IPC message arrives. + * + * There is at most one helper fibril per thread. + * + */ +static errno_t _helper_fibril_fn(void *arg) +{ + /* Set itself as the thread's own context. */ + fibril_self()->thread_ctx = fibril_self(); + + (void) arg; + + struct timeval next_timeout; + while (true) { + struct timeval *to = _handle_expired_timeouts(&next_timeout); + fibril_t *f = _ready_list_pop(to, false); + if (f) { + _fibril_switch_to(SWITCH_FROM_HELPER, f, false); } - fibril_teardown(srcf->clean_after_me, true); - srcf->clean_after_me = NULL; - } - - return 1; + } + + return EOK; } @@ -247,11 +526,11 @@ return 0; - size_t stack_size = (stksz == FIBRIL_DFLT_STK_SIZE) ? + fibril->stack_size = (stksz == FIBRIL_DFLT_STK_SIZE) ? stack_size_get() : stksz; - fibril->stack = as_area_create(AS_AREA_ANY, stack_size, + fibril->stack = as_area_create(AS_AREA_ANY, fibril->stack_size, AS_AREA_READ | AS_AREA_WRITE | AS_AREA_CACHEABLE | AS_AREA_GUARD | AS_AREA_LATE_RESERVE, AS_AREA_UNPAGED); if (fibril->stack == AS_MAP_FAILED) { - fibril_teardown(fibril, false); + fibril_teardown(fibril); return 0; } @@ -261,7 +540,7 @@ context_create_t sctx = { - .fn = fibril_main, + .fn = _fibril_main, .stack_base = fibril->stack, - .stack_size = stack_size, + .stack_size = fibril->stack_size, .tls = fibril->tcb, }; @@ -274,5 +553,5 @@ * * Free resources of a fibril that has been created with fibril_create() - * but never readied using fibril_add_ready(). + * but never started using fibril_start(). * * @param fid Pointer to the fibril structure of the fibril to be @@ -283,47 +562,152 @@ fibril_t *fibril = (fibril_t *) fid; + assert(!fibril->is_running); + assert(fibril->stack); as_area_destroy(fibril->stack); - fibril_teardown(fibril, false); -} - -/** Add a fibril to the ready list. - * - * @param fid Pointer to the fibril structure of the fibril to be - * added. - * - */ -void fibril_add_ready(fid_t fid) -{ - fibril_t *fibril = (fibril_t *) fid; + fibril_teardown(fibril); +} + +static void _insert_timeout(_timeout_t *timeout) +{ + futex_assert_is_locked(&fibril_futex); + assert(timeout); + + link_t *tmp = timeout_list.head.next; + while (tmp != &timeout_list.head) { + _timeout_t *cur = list_get_instance(tmp, _timeout_t, link); + + if (tv_gteq(&cur->expires, &timeout->expires)) + break; + + tmp = tmp->next; + } + + list_insert_before(&timeout->link, tmp); +} + +/** + * Same as `fibril_wait_for()`, except with a timeout. + * + * It is guaranteed that timing out cannot cause another thread's + * `fibril_notify()` to be lost. I.e. the function returns success if and + * only if `fibril_notify()` was called after the last call to + * wait/wait_timeout returned, and before the call timed out. + * + * @return ETIMEOUT if timed out. EOK otherwise. + */ +errno_t fibril_wait_timeout(fibril_event_t *event, const struct timeval *expires) +{ + DPRINTF("### Fibril %p sleeping on event %p.\n", fibril_self(), event); + + if (!fibril_self()->thread_ctx) { + fibril_self()->thread_ctx = + fibril_create_generic(_helper_fibril_fn, NULL, PAGE_SIZE); + if (!fibril_self()->thread_ctx) + return ENOMEM; + } futex_lock(&fibril_futex); - list_append(&fibril->link, &ready_list); + + if (event->fibril == _EVENT_TRIGGERED) { + DPRINTF("### Already triggered. Returning. \n"); + event->fibril = _EVENT_INITIAL; + futex_unlock(&fibril_futex); + return EOK; + } + + assert(event->fibril == _EVENT_INITIAL); + + fibril_t *srcf = fibril_self(); + fibril_t *dstf = NULL; + + /* + * We cannot block here waiting for another fibril becoming + * ready, since that would require unlocking the fibril_futex, + * and that in turn would allow another thread to restore + * the source fibril before this thread finished switching. + * + * Instead, we switch to an internal "helper" fibril whose only + * job is to wait for an event, freeing the source fibril for + * wakeups. There is always one for each running thread. + */ + + dstf = _ready_list_pop_nonblocking(true); + if (!dstf) { + // XXX: It is possible for the _ready_list_pop_nonblocking() to + // check for IPC, find a pending message, and trigger the + // event on which we are currently trying to sleep. + if (event->fibril == _EVENT_TRIGGERED) { + event->fibril = _EVENT_INITIAL; + futex_unlock(&fibril_futex); + return EOK; + } + + dstf = srcf->thread_ctx; + assert(dstf); + } + + _timeout_t timeout = { 0 }; + if (expires) { + timeout.expires = *expires; + timeout.event = event; + _insert_timeout(&timeout); + } + + assert(srcf); + + event->fibril = srcf; + srcf->sleep_event = event; + + assert(event->fibril != _EVENT_INITIAL); + + _fibril_switch_to(SWITCH_FROM_BLOCKED, dstf, true); + + assert(event->fibril != srcf); + assert(event->fibril != _EVENT_INITIAL); + assert(event->fibril == _EVENT_TIMED_OUT || event->fibril == _EVENT_TRIGGERED); + + list_remove(&timeout.link); + errno_t rc = (event->fibril == _EVENT_TIMED_OUT) ? ETIMEOUT : EOK; + event->fibril = _EVENT_INITIAL; + futex_unlock(&fibril_futex); -} - -/** Add a fibril to the manager list. - * - * @param fid Pointer to the fibril structure of the fibril to be - * added. - * - */ -void fibril_add_manager(fid_t fid) -{ - fibril_t *fibril = (fibril_t *) fid; - + _fibril_cleanup_dead(); + return rc; +} + +void fibril_wait_for(fibril_event_t *event) +{ + (void) fibril_wait_timeout(event, NULL); +} + +void fibril_notify(fibril_event_t *event) +{ futex_lock(&fibril_futex); - list_append(&fibril->link, &manager_list); + _ready_list_push(_fibril_trigger_internal(event, _EVENT_TRIGGERED)); futex_unlock(&fibril_futex); } -/** Remove one manager from the manager list. */ -void fibril_remove_manager(void) +/** Start a fibril that has not been running yet. */ +void fibril_start(fibril_t *fibril) { futex_lock(&fibril_futex); - if (!list_empty(&manager_list)) - list_remove(list_first(&manager_list)); + assert(!fibril->is_running); + fibril->is_running = true; + + if (!link_in_use(&fibril->all_link)) + list_append(&fibril->all_link, &fibril_list); + + _ready_list_push(fibril); + futex_unlock(&fibril_futex); } +/** Start a fibril that has not been running yet. (obsolete) */ +void fibril_add_ready(fibril_t *fibril) +{ + fibril_start(fibril); +} + +/** @return the currently running fibril. */ fibril_t *fibril_self(void) { @@ -334,8 +718,8 @@ } -/** Return fibril id of the currently running fibril. - * - * @return fibril ID of the currently running fibril. - * +/** + * Obsolete, use fibril_self(). + * + * @return ID of the currently running fibril. */ fid_t fibril_get_id(void) @@ -344,16 +728,18 @@ } +/** + * Switch to another fibril, if one is ready to run. + * Has no effect on a heavy fibril. + */ void fibril_yield(void) { - futex_lock(&async_futex); - (void) fibril_switch(FIBRIL_PREEMPT); - futex_unlock(&async_futex); + fibril_t *f = _ready_list_pop_nonblocking(false); + if (f) + _fibril_switch_to(SWITCH_FROM_YIELD, f, false); } static void _runner_fn(void *arg) { - futex_lock(&async_futex); - (void) fibril_switch(FIBRIL_FROM_BLOCKED); - __builtin_unreachable(); + _helper_fibril_fn(arg); } @@ -368,4 +754,7 @@ int fibril_test_spawn_runners(int n) { + if (!multithreaded) + multithreaded = true; + errno_t rc; @@ -394,5 +783,7 @@ // TODO: Implement better. // For now, 4 total runners is a sensible default. - fibril_test_spawn_runners(3); + if (!multithreaded) { + fibril_test_spawn_runners(3); + } } @@ -407,4 +798,71 @@ } +/** + * Exit a fibril. Never returns. + * + * @param retval Value to return from fibril_join() called on this fibril. + */ +_Noreturn void fibril_exit(long retval) +{ + // TODO: implement fibril_join() and remember retval + (void) retval; + + fibril_t *f = _ready_list_pop_nonblocking(false); + if (!f) + f = fibril_self()->thread_ctx; + + _fibril_switch_to(SWITCH_FROM_DEAD, f, false); + __builtin_unreachable(); +} + +void __fibrils_init(void) +{ + /* + * We allow a fixed, small amount of parallelism for IPC reads, but + * since IPC is currently serialized in kernel, there's not much + * we can get from more threads reading messages. + */ + +#define IPC_BUFFER_COUNT 1024 + static _ipc_buffer_t buffers[IPC_BUFFER_COUNT]; + + for (int i = 0; i < IPC_BUFFER_COUNT; i++) { + list_append(&buffers[i].link, &ipc_buffer_free_list); + futex_up(&ready_semaphore); + } +} + +void fibril_usleep(suseconds_t timeout) +{ + struct timeval expires; + getuptime(&expires); + tv_add_diff(&expires, timeout); + + fibril_event_t event = FIBRIL_EVENT_INIT; + fibril_wait_timeout(&event, &expires); +} + +void fibril_sleep(unsigned int sec) +{ + struct timeval expires; + getuptime(&expires); + expires.tv_sec += sec; + + fibril_event_t event = FIBRIL_EVENT_INIT; + fibril_wait_timeout(&event, &expires); +} + +void fibril_ipc_poke(void) +{ + DPRINTF("Poking.\n"); + /* Wakeup one thread sleeping in SYS_IPC_WAIT. */ + ipc_poke(); +} + +errno_t fibril_ipc_wait(ipc_call_t *call, const struct timeval *expires) +{ + return _wait_ipc(call, expires); +} + /** @} */ Index: uspace/lib/c/generic/fibril_synch.c =================================================================== --- uspace/lib/c/generic/fibril_synch.c (revision 7137f74c74c9c2eea4980d18b86a6d9667725946) +++ uspace/lib/c/generic/fibril_synch.c (revision 514d56134040fa72282251a1c8434e3190f9d1b6) @@ -45,4 +45,6 @@ #include #include +#include +#include #include "private/async.h" @@ -51,18 +53,17 @@ static fibril_local bool deadlocked = false; -static void optimize_execution_power(void) -{ - /* - * When waking up a worker fibril previously blocked in fibril - * synchronization, chances are that there is an idle manager fibril - * waiting for IPC, that could start executing the awakened worker - * fibril right away. We try to detect this and bring the manager - * fibril back to fruitful work. - */ - async_poke(); -} +typedef struct { + link_t link; + fibril_event_t event; + fibril_mutex_t *mutex; + fid_t fid; +} awaiter_t; + +#define AWAITER_INIT { .fid = fibril_get_id() } static void print_deadlock(fibril_owner_info_t *oi) { + // FIXME: Print to stderr. + fibril_t *f = (fibril_t *) fibril_get_id(); @@ -93,8 +94,11 @@ -static void check_for_deadlock(fibril_owner_info_t *oi) -{ +static void check_fibril_for_deadlock(fibril_owner_info_t *oi, fibril_t *fib) +{ + futex_assert_is_locked(&async_futex); + while (oi && oi->owned_by) { - if (oi->owned_by == (fibril_t *) fibril_get_id()) { + if (oi->owned_by == fib) { + futex_unlock(&async_futex); print_deadlock(oi); abort(); @@ -104,4 +108,8 @@ } +static void check_for_deadlock(fibril_owner_info_t *oi) +{ + check_fibril_for_deadlock(oi, fibril_self()); +} void fibril_mutex_initialize(fibril_mutex_t *fm) @@ -117,18 +125,19 @@ futex_lock(&async_futex); - if (fm->counter-- <= 0) { - awaiter_t wdata; - - awaiter_initialize(&wdata); - wdata.fid = fibril_get_id(); - wdata.wu_event.inlist = true; - list_append(&wdata.wu_event.link, &fm->waiters); - check_for_deadlock(&fm->oi); - f->waits_for = &fm->oi; - fibril_switch(FIBRIL_FROM_BLOCKED); - } else { + + if (fm->counter-- > 0) { fm->oi.owned_by = f; - } - futex_unlock(&async_futex); + futex_unlock(&async_futex); + return; + } + + awaiter_t wdata = AWAITER_INIT; + list_append(&wdata.link, &fm->waiters); + check_for_deadlock(&fm->oi); + f->waits_for = &fm->oi; + + futex_unlock(&async_futex); + + fibril_wait_for(&wdata.event); } @@ -150,22 +159,15 @@ static void _fibril_mutex_unlock_unsafe(fibril_mutex_t *fm) { + assert(fm->oi.owned_by == (fibril_t *) fibril_get_id()); + if (fm->counter++ < 0) { - link_t *tmp; - awaiter_t *wdp; - fibril_t *f; - - tmp = list_first(&fm->waiters); - assert(tmp != NULL); - wdp = list_get_instance(tmp, awaiter_t, wu_event.link); - wdp->active = true; - wdp->wu_event.inlist = false; - - f = (fibril_t *) wdp->fid; + awaiter_t *wdp = list_pop(&fm->waiters, awaiter_t, link); + assert(wdp); + + fibril_t *f = (fibril_t *) wdp->fid; fm->oi.owned_by = f; f->waits_for = NULL; - list_remove(&wdp->wu_event.link); - fibril_add_ready(wdp->fid); - optimize_execution_power(); + fibril_notify(&wdp->event); } else { fm->oi.owned_by = NULL; @@ -175,5 +177,4 @@ void fibril_mutex_unlock(fibril_mutex_t *fm) { - assert(fibril_mutex_is_locked(fm)); futex_lock(&async_futex); _fibril_mutex_unlock_unsafe(fm); @@ -183,11 +184,7 @@ bool fibril_mutex_is_locked(fibril_mutex_t *fm) { - bool locked = false; - - futex_lock(&async_futex); - if (fm->counter <= 0) - locked = true; - futex_unlock(&async_futex); - + futex_lock(&async_futex); + bool locked = (fm->oi.owned_by == (fibril_t *) fibril_get_id()); + futex_unlock(&async_futex); return locked; } @@ -206,21 +203,23 @@ futex_lock(&async_futex); - if (frw->writers) { - awaiter_t wdata; - - awaiter_initialize(&wdata); - wdata.fid = (fid_t) f; - wdata.wu_event.inlist = true; - f->is_writer = false; - list_append(&wdata.wu_event.link, &frw->waiters); - check_for_deadlock(&frw->oi); - f->waits_for = &frw->oi; - fibril_switch(FIBRIL_FROM_BLOCKED); - } else { + + if (!frw->writers) { /* Consider the first reader the owner. */ if (frw->readers++ == 0) frw->oi.owned_by = f; - } - futex_unlock(&async_futex); + futex_unlock(&async_futex); + return; + } + + f->is_writer = false; + + awaiter_t wdata = AWAITER_INIT; + list_append(&wdata.link, &frw->waiters); + check_for_deadlock(&frw->oi); + f->waits_for = &frw->oi; + + futex_unlock(&async_futex); + + fibril_wait_for(&wdata.event); } @@ -230,30 +229,31 @@ futex_lock(&async_futex); - if (frw->writers || frw->readers) { - awaiter_t wdata; - - awaiter_initialize(&wdata); - wdata.fid = (fid_t) f; - wdata.wu_event.inlist = true; - f->is_writer = true; - list_append(&wdata.wu_event.link, &frw->waiters); - check_for_deadlock(&frw->oi); - f->waits_for = &frw->oi; - fibril_switch(FIBRIL_FROM_BLOCKED); - } else { + + if (!frw->writers && !frw->readers) { frw->oi.owned_by = f; frw->writers++; - } - futex_unlock(&async_futex); + futex_unlock(&async_futex); + return; + } + + f->is_writer = true; + + awaiter_t wdata = AWAITER_INIT; + list_append(&wdata.link, &frw->waiters); + check_for_deadlock(&frw->oi); + f->waits_for = &frw->oi; + + futex_unlock(&async_futex); + + fibril_wait_for(&wdata.event); } static void _fibril_rwlock_common_unlock(fibril_rwlock_t *frw) { - futex_lock(&async_futex); if (frw->readers) { if (--frw->readers) { if (frw->oi.owned_by == (fibril_t *) fibril_get_id()) { /* - * If this reader firbril was considered the + * If this reader fibril was considered the * owner of this rwlock, clear the ownership * information even if there are still more @@ -267,5 +267,6 @@ frw->oi.owned_by = NULL; } - goto out; + + return; } } else { @@ -282,57 +283,47 @@ fibril_t *f; - wdp = list_get_instance(tmp, awaiter_t, wu_event.link); + wdp = list_get_instance(tmp, awaiter_t, link); f = (fibril_t *) wdp->fid; - - f->waits_for = NULL; if (f->is_writer) { if (frw->readers) break; - wdp->active = true; - wdp->wu_event.inlist = false; - list_remove(&wdp->wu_event.link); - fibril_add_ready(wdp->fid); frw->writers++; - frw->oi.owned_by = f; - optimize_execution_power(); + } else { + frw->readers++; + } + + f->waits_for = NULL; + list_remove(&wdp->link); + frw->oi.owned_by = f; + fibril_notify(&wdp->event); + + if (frw->writers) break; - } else { - wdp->active = true; - wdp->wu_event.inlist = false; - list_remove(&wdp->wu_event.link); - fibril_add_ready(wdp->fid); - if (frw->readers++ == 0) { - /* Consider the first reader the owner. */ - frw->oi.owned_by = f; - } - optimize_execution_power(); - } - } -out: - futex_unlock(&async_futex); + } } void fibril_rwlock_read_unlock(fibril_rwlock_t *frw) { - assert(fibril_rwlock_is_read_locked(frw)); + futex_lock(&async_futex); + assert(frw->readers > 0); _fibril_rwlock_common_unlock(frw); + futex_unlock(&async_futex); } void fibril_rwlock_write_unlock(fibril_rwlock_t *frw) { - assert(fibril_rwlock_is_write_locked(frw)); + futex_lock(&async_futex); + assert(frw->writers == 1); + assert(frw->oi.owned_by == fibril_self()); _fibril_rwlock_common_unlock(frw); + futex_unlock(&async_futex); } bool fibril_rwlock_is_read_locked(fibril_rwlock_t *frw) { - bool locked = false; - - futex_lock(&async_futex); - if (frw->readers) - locked = true; - futex_unlock(&async_futex); - + futex_lock(&async_futex); + bool locked = (frw->readers > 0); + futex_unlock(&async_futex); return locked; } @@ -340,13 +331,8 @@ bool fibril_rwlock_is_write_locked(fibril_rwlock_t *frw) { - bool locked = false; - - futex_lock(&async_futex); - if (frw->writers) { - assert(frw->writers == 1); - locked = true; - } - futex_unlock(&async_futex); - + futex_lock(&async_futex); + assert(frw->writers <= 1); + bool locked = (frw->writers > 0) && (frw->oi.owned_by == fibril_self()); + futex_unlock(&async_futex); return locked; } @@ -363,10 +349,13 @@ } +/** + * FIXME: If `timeout` is negative, the function returns ETIMEOUT immediately, + * and if `timeout` is 0, the wait never times out. + * This is not consistent with other similar APIs. + */ errno_t fibril_condvar_wait_timeout(fibril_condvar_t *fcv, fibril_mutex_t *fm, suseconds_t timeout) { - awaiter_t wdata; - assert(fibril_mutex_is_locked(fm)); @@ -374,73 +363,57 @@ return ETIMEOUT; - awaiter_initialize(&wdata); - wdata.fid = fibril_get_id(); - wdata.to_event.inlist = timeout > 0; - wdata.wu_event.inlist = true; - - futex_lock(&async_futex); + awaiter_t wdata = AWAITER_INIT; + wdata.mutex = fm; + + struct timeval tv; + struct timeval *expires = NULL; if (timeout) { - getuptime(&wdata.to_event.expires); - tv_add_diff(&wdata.to_event.expires, timeout); - async_insert_timeout(&wdata); - } - list_append(&wdata.wu_event.link, &fcv->waiters); + getuptime(&tv); + tv_add_diff(&tv, timeout); + expires = &tv; + } + + futex_lock(&async_futex); _fibril_mutex_unlock_unsafe(fm); - fibril_switch(FIBRIL_FROM_BLOCKED); - futex_unlock(&async_futex); - - // XXX: This could be replaced with an unlocked version to get rid - // of the unlock-lock pair. I deliberately don't do that because - // further changes would most likely need to revert that optimization. + list_append(&wdata.link, &fcv->waiters); + futex_unlock(&async_futex); + + (void) fibril_wait_timeout(&wdata.event, expires); + + futex_lock(&async_futex); + bool timed_out = link_in_use(&wdata.link); + list_remove(&wdata.link); + futex_unlock(&async_futex); + fibril_mutex_lock(fm); - futex_lock(&async_futex); - if (wdata.to_event.inlist) - list_remove(&wdata.to_event.link); - if (wdata.wu_event.inlist) - list_remove(&wdata.wu_event.link); - futex_unlock(&async_futex); - - return wdata.to_event.occurred ? ETIMEOUT : EOK; + return timed_out ? ETIMEOUT : EOK; } void fibril_condvar_wait(fibril_condvar_t *fcv, fibril_mutex_t *fm) { - errno_t rc; - - rc = fibril_condvar_wait_timeout(fcv, fm, 0); - assert(rc == EOK); -} - -static void _fibril_condvar_wakeup_common(fibril_condvar_t *fcv, bool once) -{ - link_t *tmp; - awaiter_t *wdp; - - futex_lock(&async_futex); - while (!list_empty(&fcv->waiters)) { - tmp = list_first(&fcv->waiters); - wdp = list_get_instance(tmp, awaiter_t, wu_event.link); - list_remove(&wdp->wu_event.link); - wdp->wu_event.inlist = false; - if (!wdp->active) { - wdp->active = true; - fibril_add_ready(wdp->fid); - optimize_execution_power(); - if (once) - break; - } - } - futex_unlock(&async_futex); + (void) fibril_condvar_wait_timeout(fcv, fm, 0); } void fibril_condvar_signal(fibril_condvar_t *fcv) { - _fibril_condvar_wakeup_common(fcv, true); + futex_lock(&async_futex); + + awaiter_t *w = list_pop(&fcv->waiters, awaiter_t, link); + if (w != NULL) + fibril_notify(&w->event); + + futex_unlock(&async_futex); } void fibril_condvar_broadcast(fibril_condvar_t *fcv) { - _fibril_condvar_wakeup_common(fcv, false); + futex_lock(&async_futex); + + awaiter_t *w; + while ((w = list_pop(&fcv->waiters, awaiter_t, link))) + fibril_notify(&w->event); + + futex_unlock(&async_futex); } @@ -624,5 +597,5 @@ printf("Deadlock detected.\n"); stacktrace_print(); - printf("Fibril %zx is trying to clear timer %p from " + printf("Fibril %p is trying to clear timer %p from " "inside its handler %p.\n", fibril_get_id(), timer, timer->fun); @@ -674,18 +647,11 @@ sem->count++; - if (sem->count > 0) { - futex_unlock(&async_futex); - return; - } - - link_t *tmp = list_first(&sem->waiters); - assert(tmp); - list_remove(tmp); - - futex_unlock(&async_futex); - - awaiter_t *wdp = list_get_instance(tmp, awaiter_t, wu_event.link); - fibril_add_ready(wdp->fid); - optimize_execution_power(); + if (sem->count <= 0) { + awaiter_t *w = list_pop(&sem->waiters, awaiter_t, link); + assert(w); + fibril_notify(&w->event); + } + + futex_unlock(&async_futex); } @@ -707,12 +673,10 @@ } - awaiter_t wdata; - awaiter_initialize(&wdata); - - wdata.fid = fibril_get_id(); - list_append(&wdata.wu_event.link, &sem->waiters); - - fibril_switch(FIBRIL_FROM_BLOCKED); - futex_unlock(&async_futex); + awaiter_t wdata = AWAITER_INIT; + list_append(&wdata.link, &sem->waiters); + + futex_unlock(&async_futex); + + fibril_wait_for(&wdata.event); } Index: uspace/lib/c/generic/libc.c =================================================================== --- uspace/lib/c/generic/libc.c (revision 7137f74c74c9c2eea4980d18b86a6d9667725946) +++ uspace/lib/c/generic/libc.c (revision 514d56134040fa72282251a1c8434e3190f9d1b6) @@ -86,4 +86,6 @@ assert(main_fibril.tcb); + + __fibrils_init(); /* Initialize the fibril. */ Index: uspace/lib/c/generic/private/async.h =================================================================== --- uspace/lib/c/generic/private/async.h (revision 7137f74c74c9c2eea4980d18b86a6d9667725946) +++ uspace/lib/c/generic/private/async.h (revision 514d56134040fa72282251a1c8434e3190f9d1b6) @@ -42,42 +42,4 @@ #include #include - -/** Structures of this type are used to track the timeout events. */ -typedef struct { - /** If true, this struct is in the timeout list. */ - bool inlist; - - /** Timeout list link. */ - link_t link; - - /** If true, we have timed out. */ - bool occurred; - - /** Expiration time. */ - struct timeval expires; -} to_event_t; - -/** Structures of this type are used to track the wakeup events. */ -typedef struct { - /** If true, this struct is in a synchronization object wait queue. */ - bool inlist; - - /** Wait queue linkage. */ - link_t link; -} wu_event_t; - -/** Structures of this type represent a waiting fibril. */ -typedef struct { - /** Identification of and link to the waiting fibril. */ - fid_t fid; - - /** If true, this fibril is currently active. */ - bool active; - - /** Timeout wait data. */ - to_event_t to_event; - /** Wakeup wait data. */ - wu_event_t wu_event; -} awaiter_t; /** Session data */ @@ -132,10 +94,7 @@ }; -extern void awaiter_initialize(awaiter_t *); - extern void __async_server_init(void); extern void __async_client_init(void); extern void __async_ports_init(void); -extern void async_insert_timeout(awaiter_t *); extern errno_t async_create_port_internal(iface_t, async_port_handler_t, Index: uspace/lib/c/generic/private/fibril.h =================================================================== --- uspace/lib/c/generic/private/fibril.h (revision 7137f74c74c9c2eea4980d18b86a6d9667725946) +++ uspace/lib/c/generic/private/fibril.h (revision 514d56134040fa72282251a1c8434e3190f9d1b6) @@ -42,6 +42,8 @@ context_t ctx; + uspace_arg_t uarg; link_t link; void *stack; + size_t stack_size; void *arg; errno_t (*func)(void *); @@ -51,26 +53,23 @@ errno_t retval; - fibril_owner_info_t *waits_for; + fibril_t *thread_ctx; - atomic_t futex_locks; + bool is_running : 1; bool is_writer : 1; /* In some places, we use fibril structs that can't be freed. */ bool is_freeable : 1; + + /* Debugging stuff. */ + atomic_t futex_locks; + fibril_owner_info_t *waits_for; + fibril_event_t *sleep_event; }; - -typedef enum { - FIBRIL_PREEMPT, - FIBRIL_FROM_BLOCKED, - FIBRIL_FROM_MANAGER, - FIBRIL_FROM_DEAD -} fibril_switch_type_t; extern fibril_t *fibril_alloc(void); extern void fibril_setup(fibril_t *); -extern void fibril_teardown(fibril_t *f, bool locked); -extern int fibril_switch(fibril_switch_type_t stype); -extern void fibril_add_manager(fid_t fid); -extern void fibril_remove_manager(void); +extern void fibril_teardown(fibril_t *f); extern fibril_t *fibril_self(void); +extern void __fibrils_init(void); + #endif Index: uspace/lib/c/generic/rcu.c =================================================================== --- uspace/lib/c/generic/rcu.c (revision 7137f74c74c9c2eea4980d18b86a6d9667725946) +++ uspace/lib/c/generic/rcu.c (revision 514d56134040fa72282251a1c8434e3190f9d1b6) @@ -115,5 +115,5 @@ typedef struct blocked_fibril { - fid_t id; + fibril_event_t unblock; link_t link; bool is_ready; @@ -218,5 +218,5 @@ } -/** Delimits the start of an RCU reader critical section. */ +/** Delimits the end of an RCU reader critical section. */ void rcu_read_unlock(void) { @@ -361,5 +361,5 @@ if (rcu.sync_lock.locked) { blocked_fibril_t blocked_fib; - blocked_fib.id = fibril_get_id(); + blocked_fib.unblock = FIBRIL_EVENT_INIT; list_append(&blocked_fib.link, &rcu.sync_lock.blocked_fibrils); @@ -368,7 +368,5 @@ blocked_fib.is_ready = false; fibril_rmutex_unlock(&rcu.sync_lock.mutex); - futex_lock(&async_futex); - fibril_switch(FIBRIL_FROM_BLOCKED); - futex_unlock(&async_futex); + fibril_wait_for(&blocked_fib.unblock); fibril_rmutex_lock(&rcu.sync_lock.mutex); } while (rcu.sync_lock.locked); @@ -393,5 +391,5 @@ if (!blocked_fib->is_ready) { blocked_fib->is_ready = true; - fibril_add_ready(blocked_fib->id); + fibril_notify(&blocked_fib->unblock); } } Index: uspace/lib/c/generic/thread.c =================================================================== --- uspace/lib/c/generic/thread.c (revision 7137f74c74c9c2eea4980d18b86a6d9667725946) +++ uspace/lib/c/generic/thread.c (revision 514d56134040fa72282251a1c8434e3190f9d1b6) @@ -73,9 +73,5 @@ */ - /* If there is a manager, destroy it */ - async_destroy_manager(); - - fibril_teardown(fibril, false); - + fibril_teardown(fibril); thread_exit(0); } @@ -111,5 +107,5 @@ AS_AREA_LATE_RESERVE, AS_AREA_UNPAGED); if (stack == AS_MAP_FAILED) { - fibril_teardown(fibril, false); + fibril_teardown(fibril); free(uarg); return ENOMEM; Index: uspace/lib/c/include/async.h =================================================================== --- uspace/lib/c/include/async.h (revision 7137f74c74c9c2eea4980d18b86a6d9667725946) +++ uspace/lib/c/include/async.h (revision 514d56134040fa72282251a1c8434e3190f9d1b6) @@ -143,7 +143,4 @@ extern errno_t async_wait_timeout(aid_t, errno_t *, suseconds_t); extern void async_forget(aid_t); - -extern void async_create_manager(void); -extern void async_destroy_manager(void); extern void async_set_client_data_constructor(async_client_data_ctor_t); @@ -345,5 +342,4 @@ extern errno_t async_hangup(async_sess_t *); -extern void async_poke(void); extern async_exch_t *async_exchange_begin(async_sess_t *); @@ -475,4 +471,5 @@ errno_t async_spawn_notification_handler(void); +fid_t async_create_manager(void); #endif Index: uspace/lib/c/include/fibril.h =================================================================== --- uspace/lib/c/include/fibril.h (revision 7137f74c74c9c2eea4980d18b86a6d9667725946) +++ uspace/lib/c/include/fibril.h (revision 514d56134040fa72282251a1c8434e3190f9d1b6) @@ -38,4 +38,6 @@ #include #include +#include <_bits/__noreturn.h> +#include typedef struct fibril fibril_t; @@ -45,5 +47,11 @@ } fibril_owner_info_t; -typedef sysarg_t fid_t; +typedef fibril_t *fid_t; + +typedef struct { + fibril_t *fibril; +} fibril_event_t; + +#define FIBRIL_EVENT_INIT ((fibril_event_t) {0}) /** Fibril-local variable specifier */ @@ -52,7 +60,7 @@ #define FIBRIL_DFLT_STK_SIZE 0 -extern fid_t fibril_create_generic(errno_t (*func)(void *), void *arg, size_t); -extern void fibril_destroy(fid_t fid); -extern void fibril_add_ready(fid_t fid); +extern fid_t fibril_create_generic(errno_t (*)(void *), void *, size_t); +extern void fibril_destroy(fid_t); +extern void fibril_add_ready(fid_t); extern fid_t fibril_get_id(void); extern void fibril_yield(void); @@ -71,4 +79,14 @@ } +extern void fibril_start(fid_t); +extern __noreturn void fibril_exit(long); + +extern void fibril_wait_for(fibril_event_t *); +extern errno_t fibril_wait_timeout(fibril_event_t *, const struct timeval *); +extern void fibril_notify(fibril_event_t *); + +extern errno_t fibril_ipc_wait(ipc_call_t *, const struct timeval *); +extern void fibril_ipc_poke(void); + #endif