[bc1f1c2] | 1 | /*
|
---|
| 2 | * Copyright (c) 2006 Ondrej Palkovsky
|
---|
| 3 | * Copyright (c) 2007 Jakub Jermar
|
---|
[514d561] | 4 | * Copyright (c) 2018 CZ.NIC, z.s.p.o.
|
---|
[bc1f1c2] | 5 | * All rights reserved.
|
---|
| 6 | *
|
---|
| 7 | * Redistribution and use in source and binary forms, with or without
|
---|
| 8 | * modification, are permitted provided that the following conditions
|
---|
| 9 | * are met:
|
---|
| 10 | *
|
---|
| 11 | * - Redistributions of source code must retain the above copyright
|
---|
| 12 | * notice, this list of conditions and the following disclaimer.
|
---|
| 13 | * - Redistributions in binary form must reproduce the above copyright
|
---|
| 14 | * notice, this list of conditions and the following disclaimer in the
|
---|
| 15 | * documentation and/or other materials provided with the distribution.
|
---|
| 16 | * - The name of the author may not be used to endorse or promote products
|
---|
| 17 | * derived from this software without specific prior written permission.
|
---|
| 18 | *
|
---|
| 19 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
---|
| 20 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
---|
| 21 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
---|
| 22 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
---|
| 23 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
---|
| 24 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
---|
| 25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
---|
| 26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
---|
| 27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
---|
| 28 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
---|
| 29 | */
|
---|
| 30 |
|
---|
| 31 | /** @addtogroup libc
|
---|
| 32 | * @{
|
---|
| 33 | */
|
---|
| 34 | /** @file
|
---|
| 35 | */
|
---|
| 36 |
|
---|
[d9c8c81] | 37 | #include <adt/list.h>
|
---|
[bc1f1c2] | 38 | #include <fibril.h>
|
---|
[0aae87a6] | 39 | #include <stack.h>
|
---|
[fa23560] | 40 | #include <tls.h>
|
---|
[38d150e] | 41 | #include <stdlib.h>
|
---|
[1107050] | 42 | #include <as.h>
|
---|
[e0a4686] | 43 | #include <context.h>
|
---|
[bc1f1c2] | 44 | #include <futex.h>
|
---|
| 45 | #include <assert.h>
|
---|
| 46 |
|
---|
[514d561] | 47 | #include <mem.h>
|
---|
| 48 | #include <str.h>
|
---|
| 49 | #include <ipc/ipc.h>
|
---|
| 50 | #include <libarch/faddr.h>
|
---|
[38d8849] | 51 | #include "private/thread.h"
|
---|
[d73d992] | 52 | #include "private/fibril.h"
|
---|
[40abf56] | 53 | #include "private/libc.h"
|
---|
[bc1f1c2] | 54 |
|
---|
[514d561] | 55 | #define DPRINTF(...) ((void)0)
|
---|
| 56 |
|
---|
| 57 | /** Member of timeout_list. */
|
---|
| 58 | typedef struct {
|
---|
| 59 | link_t link;
|
---|
| 60 | struct timeval expires;
|
---|
| 61 | fibril_event_t *event;
|
---|
| 62 | } _timeout_t;
|
---|
| 63 |
|
---|
| 64 | typedef struct {
|
---|
| 65 | errno_t rc;
|
---|
| 66 | link_t link;
|
---|
| 67 | ipc_call_t *call;
|
---|
| 68 | fibril_event_t event;
|
---|
| 69 | } _ipc_waiter_t;
|
---|
| 70 |
|
---|
| 71 | typedef struct {
|
---|
| 72 | errno_t rc;
|
---|
| 73 | link_t link;
|
---|
| 74 | ipc_call_t call;
|
---|
| 75 | } _ipc_buffer_t;
|
---|
| 76 |
|
---|
| 77 | typedef enum {
|
---|
| 78 | SWITCH_FROM_DEAD,
|
---|
| 79 | SWITCH_FROM_HELPER,
|
---|
| 80 | SWITCH_FROM_YIELD,
|
---|
| 81 | SWITCH_FROM_BLOCKED,
|
---|
| 82 | } _switch_type_t;
|
---|
| 83 |
|
---|
| 84 | static bool multithreaded = false;
|
---|
| 85 |
|
---|
| 86 | /* This futex serializes access to global data. */
|
---|
[927a181e] | 87 | static futex_t fibril_futex = FUTEX_INITIALIZER;
|
---|
[514d561] | 88 | static futex_t ready_semaphore = FUTEX_INITIALIZE(0);
|
---|
[12f91130] | 89 |
|
---|
[bc1f1c2] | 90 | static LIST_INITIALIZE(ready_list);
|
---|
[c1b979a] | 91 | static LIST_INITIALIZE(fibril_list);
|
---|
[514d561] | 92 | static LIST_INITIALIZE(timeout_list);
|
---|
| 93 |
|
---|
| 94 | static futex_t ipc_lists_futex = FUTEX_INITIALIZER;
|
---|
| 95 | static LIST_INITIALIZE(ipc_waiter_list);
|
---|
| 96 | static LIST_INITIALIZE(ipc_buffer_list);
|
---|
| 97 | static LIST_INITIALIZE(ipc_buffer_free_list);
|
---|
| 98 |
|
---|
| 99 | /* Only used as unique markers for triggered events. */
|
---|
| 100 | static fibril_t _fibril_event_triggered;
|
---|
| 101 | static fibril_t _fibril_event_timed_out;
|
---|
| 102 | #define _EVENT_INITIAL (NULL)
|
---|
| 103 | #define _EVENT_TRIGGERED (&_fibril_event_triggered)
|
---|
| 104 | #define _EVENT_TIMED_OUT (&_fibril_event_timed_out)
|
---|
| 105 |
|
---|
| 106 | static atomic_t threads_in_ipc_wait = { 0 };
|
---|
[bc1f1c2] | 107 |
|
---|
[596d65c] | 108 | /** Function that spans the whole life-cycle of a fibril.
|
---|
| 109 | *
|
---|
| 110 | * Each fibril begins execution in this function. Then the function implementing
|
---|
| 111 | * the fibril logic is called. After its return, the return value is saved.
|
---|
| 112 | * The fibril then switches to another fibril, which cleans up after it.
|
---|
| 113 | *
|
---|
| 114 | */
|
---|
[514d561] | 115 | static void _fibril_main(void)
|
---|
[bc1f1c2] | 116 | {
|
---|
[514d561] | 117 | /* fibril_futex is locked when a fibril is started. */
|
---|
[899342e] | 118 | futex_unlock(&fibril_futex);
|
---|
| 119 |
|
---|
[d73d992] | 120 | fibril_t *fibril = fibril_self();
|
---|
[d54b303] | 121 |
|
---|
[596d65c] | 122 | /* Call the implementing function. */
|
---|
[514d561] | 123 | fibril_exit(fibril->func(fibril->arg));
|
---|
[a35b458] | 124 |
|
---|
[596d65c] | 125 | /* Not reached */
|
---|
| 126 | }
|
---|
[bc1f1c2] | 127 |
|
---|
[40abf56] | 128 | /** Allocate a fibril structure and TCB, but don't do anything else with it. */
|
---|
| 129 | fibril_t *fibril_alloc(void)
|
---|
[596d65c] | 130 | {
|
---|
[40abf56] | 131 | tcb_t *tcb = tls_make(__progsymbols.elfstart);
|
---|
[bc1f1c2] | 132 | if (!tcb)
|
---|
| 133 | return NULL;
|
---|
[a35b458] | 134 |
|
---|
[d73d992] | 135 | fibril_t *fibril = calloc(1, sizeof(fibril_t));
|
---|
[596d65c] | 136 | if (!fibril) {
|
---|
[31399f3] | 137 | tls_free(tcb);
|
---|
[bc1f1c2] | 138 | return NULL;
|
---|
| 139 | }
|
---|
[a35b458] | 140 |
|
---|
[596d65c] | 141 | tcb->fibril_data = fibril;
|
---|
| 142 | fibril->tcb = tcb;
|
---|
[40abf56] | 143 | fibril->is_freeable = true;
|
---|
[a35b458] | 144 |
|
---|
[40abf56] | 145 | fibril_setup(fibril);
|
---|
[596d65c] | 146 | return fibril;
|
---|
[bc1f1c2] | 147 | }
|
---|
| 148 |
|
---|
[40abf56] | 149 | /**
|
---|
| 150 | * Put the fibril into fibril_list.
|
---|
| 151 | */
|
---|
| 152 | void fibril_setup(fibril_t *f)
|
---|
| 153 | {
|
---|
| 154 | futex_lock(&fibril_futex);
|
---|
| 155 | list_append(&f->all_link, &fibril_list);
|
---|
| 156 | futex_unlock(&fibril_futex);
|
---|
| 157 | }
|
---|
| 158 |
|
---|
[514d561] | 159 | void fibril_teardown(fibril_t *fibril)
|
---|
[1b20da0] | 160 | {
|
---|
[514d561] | 161 | futex_lock(&fibril_futex);
|
---|
[c1b979a] | 162 | list_remove(&fibril->all_link);
|
---|
[514d561] | 163 | futex_unlock(&fibril_futex);
|
---|
[40abf56] | 164 |
|
---|
| 165 | if (fibril->is_freeable) {
|
---|
| 166 | tls_free(fibril->tcb);
|
---|
| 167 | free(fibril);
|
---|
| 168 | }
|
---|
[bc1f1c2] | 169 | }
|
---|
| 170 |
|
---|
[514d561] | 171 | /**
|
---|
| 172 | * Event notification with a given reason.
|
---|
[596d65c] | 173 | *
|
---|
[514d561] | 174 | * @param reason Reason of the notification.
|
---|
| 175 | * Can be either _EVENT_TRIGGERED or _EVENT_TIMED_OUT.
|
---|
[bc1f1c2] | 176 | */
|
---|
[514d561] | 177 | static fibril_t *_fibril_trigger_internal(fibril_event_t *event, fibril_t *reason)
|
---|
[bc1f1c2] | 178 | {
|
---|
[514d561] | 179 | assert(reason != _EVENT_INITIAL);
|
---|
| 180 | assert(reason == _EVENT_TIMED_OUT || reason == _EVENT_TRIGGERED);
|
---|
| 181 |
|
---|
| 182 | futex_assert_is_locked(&fibril_futex);
|
---|
| 183 |
|
---|
| 184 | if (event->fibril == _EVENT_INITIAL) {
|
---|
| 185 | event->fibril = reason;
|
---|
| 186 | return NULL;
|
---|
| 187 | }
|
---|
| 188 |
|
---|
| 189 | if (event->fibril == _EVENT_TIMED_OUT) {
|
---|
| 190 | assert(reason == _EVENT_TRIGGERED);
|
---|
| 191 | event->fibril = reason;
|
---|
| 192 | return NULL;
|
---|
| 193 | }
|
---|
| 194 |
|
---|
| 195 | if (event->fibril == _EVENT_TRIGGERED) {
|
---|
| 196 | /* Already triggered. Nothing to do. */
|
---|
| 197 | return NULL;
|
---|
| 198 | }
|
---|
| 199 |
|
---|
| 200 | fibril_t *f = event->fibril;
|
---|
| 201 | event->fibril = reason;
|
---|
| 202 |
|
---|
| 203 | assert(f->sleep_event == event);
|
---|
| 204 | return f;
|
---|
| 205 | }
|
---|
| 206 |
|
---|
| 207 | static errno_t _ipc_wait(ipc_call_t *call, const struct timeval *expires)
|
---|
| 208 | {
|
---|
| 209 | if (!expires)
|
---|
| 210 | return ipc_wait(call, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NONE);
|
---|
| 211 |
|
---|
| 212 | if (expires->tv_sec == 0)
|
---|
| 213 | return ipc_wait(call, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NON_BLOCKING);
|
---|
| 214 |
|
---|
| 215 | struct timeval now;
|
---|
| 216 | getuptime(&now);
|
---|
| 217 |
|
---|
| 218 | if (tv_gteq(&now, expires))
|
---|
| 219 | return ipc_wait(call, SYNCH_NO_TIMEOUT, SYNCH_FLAGS_NON_BLOCKING);
|
---|
| 220 |
|
---|
| 221 | return ipc_wait(call, tv_sub_diff(expires, &now), SYNCH_FLAGS_NONE);
|
---|
| 222 | }
|
---|
| 223 |
|
---|
| 224 | /*
|
---|
| 225 | * Waits until a ready fibril is added to the list, or an IPC message arrives.
|
---|
| 226 | * Returns NULL on timeout and may also return NULL if returning from IPC
|
---|
| 227 | * wait after new ready fibrils are added.
|
---|
| 228 | */
|
---|
| 229 | static fibril_t *_ready_list_pop(const struct timeval *expires, bool locked)
|
---|
| 230 | {
|
---|
| 231 | if (locked) {
|
---|
| 232 | futex_assert_is_locked(&fibril_futex);
|
---|
| 233 | assert(expires);
|
---|
| 234 | /* Must be nonblocking. */
|
---|
| 235 | assert(expires->tv_sec == 0);
|
---|
| 236 | } else {
|
---|
| 237 | futex_assert_is_not_locked(&fibril_futex);
|
---|
| 238 | }
|
---|
| 239 |
|
---|
| 240 | if (!multithreaded) {
|
---|
| 241 | /*
|
---|
| 242 | * The number of available tokens is always equal to the number
|
---|
| 243 | * of fibrils in the ready list + the number of free IPC buffer
|
---|
| 244 | * buckets.
|
---|
| 245 | */
|
---|
| 246 |
|
---|
| 247 | assert(atomic_get(&ready_semaphore.val) ==
|
---|
| 248 | list_count(&ready_list) + list_count(&ipc_buffer_free_list));
|
---|
| 249 | }
|
---|
| 250 |
|
---|
| 251 | errno_t rc = futex_down_timeout(&ready_semaphore, expires);
|
---|
| 252 |
|
---|
| 253 | if (rc != EOK)
|
---|
| 254 | return NULL;
|
---|
| 255 |
|
---|
| 256 | /*
|
---|
| 257 | * Once we acquire a token from ready_semaphore, there are two options.
|
---|
| 258 | * Either there is a ready fibril in the list, or it's our turn to
|
---|
| 259 | * call `ipc_wait_cycle()`. There is one extra token on the semaphore
|
---|
| 260 | * for each entry of the call buffer.
|
---|
| 261 | */
|
---|
| 262 |
|
---|
| 263 |
|
---|
| 264 | if (!locked)
|
---|
| 265 | futex_lock(&fibril_futex);
|
---|
| 266 | fibril_t *f = list_pop(&ready_list, fibril_t, link);
|
---|
| 267 | if (!f)
|
---|
| 268 | atomic_inc(&threads_in_ipc_wait);
|
---|
| 269 | if (!locked)
|
---|
| 270 | futex_unlock(&fibril_futex);
|
---|
| 271 |
|
---|
| 272 | if (f)
|
---|
| 273 | return f;
|
---|
| 274 |
|
---|
| 275 | if (!multithreaded)
|
---|
| 276 | assert(list_empty(&ipc_buffer_list));
|
---|
| 277 |
|
---|
| 278 | /* No fibril is ready, IPC wait it is. */
|
---|
| 279 | ipc_call_t call = { 0 };
|
---|
| 280 | rc = _ipc_wait(&call, expires);
|
---|
| 281 |
|
---|
| 282 | atomic_dec(&threads_in_ipc_wait);
|
---|
| 283 |
|
---|
| 284 | if (rc != EOK && rc != ENOENT) {
|
---|
| 285 | /* Return token. */
|
---|
| 286 | futex_up(&ready_semaphore);
|
---|
| 287 | return NULL;
|
---|
| 288 | }
|
---|
| 289 |
|
---|
| 290 | /*
|
---|
| 291 | * We might get ENOENT due to a poke.
|
---|
| 292 | * In that case, we propagate the null call out of fibril_ipc_wait(),
|
---|
| 293 | * because poke must result in that call returning.
|
---|
| 294 | */
|
---|
| 295 |
|
---|
| 296 | /*
|
---|
| 297 | * If a fibril is already waiting for IPC, we wake up the fibril,
|
---|
| 298 | * and return the token to ready_semaphore.
|
---|
| 299 | * If there is no fibril waiting, we pop a buffer bucket and
|
---|
| 300 | * put our call there. The token then returns when the bucket is
|
---|
| 301 | * returned.
|
---|
| 302 | */
|
---|
| 303 |
|
---|
| 304 | if (!locked)
|
---|
| 305 | futex_lock(&fibril_futex);
|
---|
| 306 |
|
---|
| 307 | futex_lock(&ipc_lists_futex);
|
---|
| 308 |
|
---|
| 309 |
|
---|
| 310 | _ipc_waiter_t *w = list_pop(&ipc_waiter_list, _ipc_waiter_t, link);
|
---|
| 311 | if (w) {
|
---|
| 312 | *w->call = call;
|
---|
| 313 | w->rc = rc;
|
---|
| 314 | /* We switch to the woken up fibril immediately if possible. */
|
---|
| 315 | f = _fibril_trigger_internal(&w->event, _EVENT_TRIGGERED);
|
---|
| 316 |
|
---|
| 317 | /* Return token. */
|
---|
| 318 | futex_up(&ready_semaphore);
|
---|
| 319 | } else {
|
---|
| 320 | _ipc_buffer_t *buf = list_pop(&ipc_buffer_free_list, _ipc_buffer_t, link);
|
---|
| 321 | assert(buf);
|
---|
| 322 | *buf = (_ipc_buffer_t) { .call = call, .rc = rc };
|
---|
| 323 | list_append(&buf->link, &ipc_buffer_list);
|
---|
| 324 | }
|
---|
| 325 |
|
---|
| 326 | futex_unlock(&ipc_lists_futex);
|
---|
| 327 |
|
---|
| 328 | if (!locked)
|
---|
| 329 | futex_unlock(&fibril_futex);
|
---|
| 330 |
|
---|
| 331 | return f;
|
---|
| 332 | }
|
---|
| 333 |
|
---|
| 334 | static fibril_t *_ready_list_pop_nonblocking(bool locked)
|
---|
| 335 | {
|
---|
| 336 | struct timeval tv = { .tv_sec = 0, .tv_usec = 0 };
|
---|
| 337 | return _ready_list_pop(&tv, locked);
|
---|
| 338 | }
|
---|
| 339 |
|
---|
| 340 | static void _ready_list_push(fibril_t *f)
|
---|
| 341 | {
|
---|
| 342 | if (!f)
|
---|
| 343 | return;
|
---|
| 344 |
|
---|
| 345 | futex_assert_is_locked(&fibril_futex);
|
---|
| 346 |
|
---|
| 347 | /* Enqueue in ready_list. */
|
---|
| 348 | list_append(&f->link, &ready_list);
|
---|
| 349 | futex_up(&ready_semaphore);
|
---|
| 350 |
|
---|
| 351 | if (atomic_get(&threads_in_ipc_wait)) {
|
---|
| 352 | DPRINTF("Poking.\n");
|
---|
| 353 | /* Wakeup one thread sleeping in SYS_IPC_WAIT. */
|
---|
| 354 | ipc_poke();
|
---|
| 355 | }
|
---|
| 356 | }
|
---|
| 357 |
|
---|
| 358 | /* Blocks the current fibril until an IPC call arrives. */
|
---|
| 359 | static errno_t _wait_ipc(ipc_call_t *call, const struct timeval *expires)
|
---|
| 360 | {
|
---|
| 361 | futex_assert_is_not_locked(&fibril_futex);
|
---|
| 362 |
|
---|
| 363 | futex_lock(&ipc_lists_futex);
|
---|
| 364 | _ipc_buffer_t *buf = list_pop(&ipc_buffer_list, _ipc_buffer_t, link);
|
---|
| 365 | if (buf) {
|
---|
| 366 | *call = buf->call;
|
---|
| 367 | errno_t rc = buf->rc;
|
---|
| 368 |
|
---|
| 369 | /* Return to freelist. */
|
---|
| 370 | list_append(&buf->link, &ipc_buffer_free_list);
|
---|
| 371 | /* Return IPC wait token. */
|
---|
| 372 | futex_up(&ready_semaphore);
|
---|
| 373 |
|
---|
| 374 | futex_unlock(&ipc_lists_futex);
|
---|
| 375 | return rc;
|
---|
| 376 | }
|
---|
| 377 |
|
---|
| 378 | _ipc_waiter_t w = { .call = call };
|
---|
| 379 | list_append(&w.link, &ipc_waiter_list);
|
---|
| 380 | futex_unlock(&ipc_lists_futex);
|
---|
| 381 |
|
---|
| 382 | errno_t rc = fibril_wait_timeout(&w.event, expires);
|
---|
| 383 | if (rc == EOK)
|
---|
| 384 | return w.rc;
|
---|
| 385 |
|
---|
| 386 | futex_lock(&ipc_lists_futex);
|
---|
| 387 | if (link_in_use(&w.link))
|
---|
| 388 | list_remove(&w.link);
|
---|
| 389 | else
|
---|
| 390 | rc = w.rc;
|
---|
| 391 | futex_unlock(&ipc_lists_futex);
|
---|
| 392 | return rc;
|
---|
| 393 | }
|
---|
| 394 |
|
---|
| 395 | /** Fire all timeouts that expired. */
|
---|
| 396 | static struct timeval *_handle_expired_timeouts(struct timeval *next_timeout)
|
---|
| 397 | {
|
---|
| 398 | struct timeval tv;
|
---|
| 399 | getuptime(&tv);
|
---|
[ab6edb6] | 400 |
|
---|
[df7cbc6] | 401 | futex_lock(&fibril_futex);
|
---|
[c721d26] | 402 |
|
---|
[514d561] | 403 | while (!list_empty(&timeout_list)) {
|
---|
| 404 | link_t *cur = list_first(&timeout_list);
|
---|
| 405 | _timeout_t *to = list_get_instance(cur, _timeout_t, link);
|
---|
[e0a4686] | 406 |
|
---|
[514d561] | 407 | if (tv_gt(&to->expires, &tv)) {
|
---|
| 408 | *next_timeout = to->expires;
|
---|
[ab6edb6] | 409 | futex_unlock(&fibril_futex);
|
---|
[514d561] | 410 | return next_timeout;
|
---|
[ab6edb6] | 411 | }
|
---|
[c721d26] | 412 |
|
---|
[514d561] | 413 | list_remove(&to->link);
|
---|
[a35b458] | 414 |
|
---|
[514d561] | 415 | _ready_list_push(_fibril_trigger_internal(
|
---|
| 416 | to->event, _EVENT_TIMED_OUT));
|
---|
[bc1f1c2] | 417 | }
|
---|
[ab6edb6] | 418 |
|
---|
[514d561] | 419 | futex_unlock(&fibril_futex);
|
---|
| 420 | return NULL;
|
---|
| 421 | }
|
---|
| 422 |
|
---|
| 423 | /**
|
---|
| 424 | * Clean up after a dead fibril from which we restored context, if any.
|
---|
| 425 | * Called after a switch is made and fibril_futex is unlocked.
|
---|
| 426 | */
|
---|
| 427 | static void _fibril_cleanup_dead(void)
|
---|
| 428 | {
|
---|
| 429 | fibril_t *srcf = fibril_self();
|
---|
| 430 | if (!srcf->clean_after_me)
|
---|
| 431 | return;
|
---|
| 432 |
|
---|
| 433 | void *stack = srcf->clean_after_me->stack;
|
---|
| 434 | assert(stack);
|
---|
| 435 | as_area_destroy(stack);
|
---|
| 436 | fibril_teardown(srcf->clean_after_me);
|
---|
| 437 | srcf->clean_after_me = NULL;
|
---|
| 438 | }
|
---|
| 439 |
|
---|
| 440 | /** Switch to a fibril. */
|
---|
| 441 | static void _fibril_switch_to(_switch_type_t type, fibril_t *dstf, bool locked)
|
---|
| 442 | {
|
---|
| 443 | if (!locked)
|
---|
| 444 | futex_lock(&fibril_futex);
|
---|
| 445 | else
|
---|
| 446 | futex_assert_is_locked(&fibril_futex);
|
---|
| 447 |
|
---|
| 448 | fibril_t *srcf = fibril_self();
|
---|
| 449 | assert(srcf);
|
---|
| 450 | assert(dstf);
|
---|
[a35b458] | 451 |
|
---|
[514d561] | 452 | switch (type) {
|
---|
| 453 | case SWITCH_FROM_YIELD:
|
---|
| 454 | _ready_list_push(srcf);
|
---|
[e0a4686] | 455 | break;
|
---|
[514d561] | 456 | case SWITCH_FROM_DEAD:
|
---|
| 457 | dstf->clean_after_me = srcf;
|
---|
[e0a4686] | 458 | break;
|
---|
[514d561] | 459 | case SWITCH_FROM_HELPER:
|
---|
| 460 | case SWITCH_FROM_BLOCKED:
|
---|
[e0a4686] | 461 | break;
|
---|
| 462 | }
|
---|
| 463 |
|
---|
[514d561] | 464 | dstf->thread_ctx = srcf->thread_ctx;
|
---|
| 465 | srcf->thread_ctx = NULL;
|
---|
| 466 |
|
---|
| 467 | /* Just some bookkeeping to allow better debugging of futex locks. */
|
---|
[f6372be9] | 468 | futex_give_to(&fibril_futex, dstf);
|
---|
| 469 |
|
---|
[e0a4686] | 470 | /* Swap to the next fibril. */
|
---|
| 471 | context_swap(&srcf->ctx, &dstf->ctx);
|
---|
| 472 |
|
---|
[514d561] | 473 | assert(srcf == fibril_self());
|
---|
| 474 | assert(srcf->thread_ctx);
|
---|
[e0a4686] | 475 |
|
---|
[514d561] | 476 | if (!locked) {
|
---|
| 477 | /* Must be after context_swap()! */
|
---|
| 478 | futex_unlock(&fibril_futex);
|
---|
| 479 | _fibril_cleanup_dead();
|
---|
| 480 | }
|
---|
| 481 | }
|
---|
[899342e] | 482 |
|
---|
[514d561] | 483 | /**
|
---|
| 484 | * Main function for a helper fibril.
|
---|
| 485 | * The helper fibril executes on threads in the lightweight fibril pool when
|
---|
| 486 | * there is no fibril ready to run. Its only purpose is to block until
|
---|
| 487 | * another fibril is ready, or a timeout expires, or an IPC message arrives.
|
---|
| 488 | *
|
---|
| 489 | * There is at most one helper fibril per thread.
|
---|
| 490 | *
|
---|
| 491 | */
|
---|
| 492 | static errno_t _helper_fibril_fn(void *arg)
|
---|
| 493 | {
|
---|
| 494 | /* Set itself as the thread's own context. */
|
---|
| 495 | fibril_self()->thread_ctx = fibril_self();
|
---|
| 496 |
|
---|
| 497 | (void) arg;
|
---|
| 498 |
|
---|
| 499 | struct timeval next_timeout;
|
---|
| 500 | while (true) {
|
---|
| 501 | struct timeval *to = _handle_expired_timeouts(&next_timeout);
|
---|
| 502 | fibril_t *f = _ready_list_pop(to, false);
|
---|
| 503 | if (f) {
|
---|
| 504 | _fibril_switch_to(SWITCH_FROM_HELPER, f, false);
|
---|
[e0a4686] | 505 | }
|
---|
| 506 | }
|
---|
| 507 |
|
---|
[514d561] | 508 | return EOK;
|
---|
[bc1f1c2] | 509 | }
|
---|
| 510 |
|
---|
| 511 | /** Create a new fibril.
|
---|
| 512 | *
|
---|
[596d65c] | 513 | * @param func Implementing function of the new fibril.
|
---|
| 514 | * @param arg Argument to pass to func.
|
---|
[eceff5f] | 515 | * @param stksz Stack size in bytes.
|
---|
[596d65c] | 516 | *
|
---|
| 517 | * @return 0 on failure or TLS of the new fibril.
|
---|
[bc1f1c2] | 518 | *
|
---|
| 519 | */
|
---|
[b7fd2a0] | 520 | fid_t fibril_create_generic(errno_t (*func)(void *), void *arg, size_t stksz)
|
---|
[bc1f1c2] | 521 | {
|
---|
[596d65c] | 522 | fibril_t *fibril;
|
---|
[a35b458] | 523 |
|
---|
[40abf56] | 524 | fibril = fibril_alloc();
|
---|
[596d65c] | 525 | if (fibril == NULL)
|
---|
[bc1f1c2] | 526 | return 0;
|
---|
[a35b458] | 527 |
|
---|
[514d561] | 528 | fibril->stack_size = (stksz == FIBRIL_DFLT_STK_SIZE) ?
|
---|
[eceff5f] | 529 | stack_size_get() : stksz;
|
---|
[514d561] | 530 | fibril->stack = as_area_create(AS_AREA_ANY, fibril->stack_size,
|
---|
[1107050] | 531 | AS_AREA_READ | AS_AREA_WRITE | AS_AREA_CACHEABLE | AS_AREA_GUARD |
|
---|
[6aeca0d] | 532 | AS_AREA_LATE_RESERVE, AS_AREA_UNPAGED);
|
---|
[40abf56] | 533 | if (fibril->stack == AS_MAP_FAILED) {
|
---|
[514d561] | 534 | fibril_teardown(fibril);
|
---|
[bc1f1c2] | 535 | return 0;
|
---|
| 536 | }
|
---|
[a35b458] | 537 |
|
---|
[596d65c] | 538 | fibril->func = func;
|
---|
| 539 | fibril->arg = arg;
|
---|
[7f122e3] | 540 |
|
---|
[e0a4686] | 541 | context_create_t sctx = {
|
---|
[514d561] | 542 | .fn = _fibril_main,
|
---|
[e0a4686] | 543 | .stack_base = fibril->stack,
|
---|
[514d561] | 544 | .stack_size = fibril->stack_size,
|
---|
[e0a4686] | 545 | .tls = fibril->tcb,
|
---|
| 546 | };
|
---|
[bc1f1c2] | 547 |
|
---|
[e0a4686] | 548 | context_create(&fibril->ctx, &sctx);
|
---|
[596d65c] | 549 | return (fid_t) fibril;
|
---|
[bc1f1c2] | 550 | }
|
---|
| 551 |
|
---|
[32d19f7] | 552 | /** Delete a fibril that has never run.
|
---|
| 553 | *
|
---|
| 554 | * Free resources of a fibril that has been created with fibril_create()
|
---|
[514d561] | 555 | * but never started using fibril_start().
|
---|
[32d19f7] | 556 | *
|
---|
| 557 | * @param fid Pointer to the fibril structure of the fibril to be
|
---|
| 558 | * added.
|
---|
| 559 | */
|
---|
| 560 | void fibril_destroy(fid_t fid)
|
---|
| 561 | {
|
---|
| 562 | fibril_t *fibril = (fibril_t *) fid;
|
---|
[a35b458] | 563 |
|
---|
[514d561] | 564 | assert(!fibril->is_running);
|
---|
| 565 | assert(fibril->stack);
|
---|
[1107050] | 566 | as_area_destroy(fibril->stack);
|
---|
[514d561] | 567 | fibril_teardown(fibril);
|
---|
| 568 | }
|
---|
| 569 |
|
---|
| 570 | static void _insert_timeout(_timeout_t *timeout)
|
---|
| 571 | {
|
---|
| 572 | futex_assert_is_locked(&fibril_futex);
|
---|
| 573 | assert(timeout);
|
---|
| 574 |
|
---|
| 575 | link_t *tmp = timeout_list.head.next;
|
---|
| 576 | while (tmp != &timeout_list.head) {
|
---|
| 577 | _timeout_t *cur = list_get_instance(tmp, _timeout_t, link);
|
---|
| 578 |
|
---|
| 579 | if (tv_gteq(&cur->expires, &timeout->expires))
|
---|
| 580 | break;
|
---|
| 581 |
|
---|
| 582 | tmp = tmp->next;
|
---|
| 583 | }
|
---|
| 584 |
|
---|
| 585 | list_insert_before(&timeout->link, tmp);
|
---|
[32d19f7] | 586 | }
|
---|
| 587 |
|
---|
[514d561] | 588 | /**
|
---|
| 589 | * Same as `fibril_wait_for()`, except with a timeout.
|
---|
[bc1f1c2] | 590 | *
|
---|
[514d561] | 591 | * It is guaranteed that timing out cannot cause another thread's
|
---|
| 592 | * `fibril_notify()` to be lost. I.e. the function returns success if and
|
---|
| 593 | * only if `fibril_notify()` was called after the last call to
|
---|
| 594 | * wait/wait_timeout returned, and before the call timed out.
|
---|
[596d65c] | 595 | *
|
---|
[514d561] | 596 | * @return ETIMEOUT if timed out. EOK otherwise.
|
---|
[bc1f1c2] | 597 | */
|
---|
[514d561] | 598 | errno_t fibril_wait_timeout(fibril_event_t *event, const struct timeval *expires)
|
---|
[bc1f1c2] | 599 | {
|
---|
[514d561] | 600 | DPRINTF("### Fibril %p sleeping on event %p.\n", fibril_self(), event);
|
---|
| 601 |
|
---|
| 602 | if (!fibril_self()->thread_ctx) {
|
---|
| 603 | fibril_self()->thread_ctx =
|
---|
| 604 | fibril_create_generic(_helper_fibril_fn, NULL, PAGE_SIZE);
|
---|
| 605 | if (!fibril_self()->thread_ctx)
|
---|
| 606 | return ENOMEM;
|
---|
| 607 | }
|
---|
[a35b458] | 608 |
|
---|
[df7cbc6] | 609 | futex_lock(&fibril_futex);
|
---|
[514d561] | 610 |
|
---|
| 611 | if (event->fibril == _EVENT_TRIGGERED) {
|
---|
| 612 | DPRINTF("### Already triggered. Returning. \n");
|
---|
| 613 | event->fibril = _EVENT_INITIAL;
|
---|
| 614 | futex_unlock(&fibril_futex);
|
---|
| 615 | return EOK;
|
---|
| 616 | }
|
---|
| 617 |
|
---|
| 618 | assert(event->fibril == _EVENT_INITIAL);
|
---|
| 619 |
|
---|
| 620 | fibril_t *srcf = fibril_self();
|
---|
| 621 | fibril_t *dstf = NULL;
|
---|
| 622 |
|
---|
| 623 | /*
|
---|
| 624 | * We cannot block here waiting for another fibril becoming
|
---|
| 625 | * ready, since that would require unlocking the fibril_futex,
|
---|
| 626 | * and that in turn would allow another thread to restore
|
---|
| 627 | * the source fibril before this thread finished switching.
|
---|
| 628 | *
|
---|
| 629 | * Instead, we switch to an internal "helper" fibril whose only
|
---|
| 630 | * job is to wait for an event, freeing the source fibril for
|
---|
| 631 | * wakeups. There is always one for each running thread.
|
---|
| 632 | */
|
---|
| 633 |
|
---|
| 634 | dstf = _ready_list_pop_nonblocking(true);
|
---|
| 635 | if (!dstf) {
|
---|
| 636 | // XXX: It is possible for the _ready_list_pop_nonblocking() to
|
---|
| 637 | // check for IPC, find a pending message, and trigger the
|
---|
| 638 | // event on which we are currently trying to sleep.
|
---|
| 639 | if (event->fibril == _EVENT_TRIGGERED) {
|
---|
| 640 | event->fibril = _EVENT_INITIAL;
|
---|
| 641 | futex_unlock(&fibril_futex);
|
---|
| 642 | return EOK;
|
---|
| 643 | }
|
---|
| 644 |
|
---|
| 645 | dstf = srcf->thread_ctx;
|
---|
| 646 | assert(dstf);
|
---|
| 647 | }
|
---|
| 648 |
|
---|
| 649 | _timeout_t timeout = { 0 };
|
---|
| 650 | if (expires) {
|
---|
| 651 | timeout.expires = *expires;
|
---|
| 652 | timeout.event = event;
|
---|
| 653 | _insert_timeout(&timeout);
|
---|
| 654 | }
|
---|
| 655 |
|
---|
| 656 | assert(srcf);
|
---|
| 657 |
|
---|
| 658 | event->fibril = srcf;
|
---|
| 659 | srcf->sleep_event = event;
|
---|
| 660 |
|
---|
| 661 | assert(event->fibril != _EVENT_INITIAL);
|
---|
| 662 |
|
---|
| 663 | _fibril_switch_to(SWITCH_FROM_BLOCKED, dstf, true);
|
---|
| 664 |
|
---|
| 665 | assert(event->fibril != srcf);
|
---|
| 666 | assert(event->fibril != _EVENT_INITIAL);
|
---|
| 667 | assert(event->fibril == _EVENT_TIMED_OUT || event->fibril == _EVENT_TRIGGERED);
|
---|
| 668 |
|
---|
| 669 | list_remove(&timeout.link);
|
---|
| 670 | errno_t rc = (event->fibril == _EVENT_TIMED_OUT) ? ETIMEOUT : EOK;
|
---|
| 671 | event->fibril = _EVENT_INITIAL;
|
---|
| 672 |
|
---|
[df7cbc6] | 673 | futex_unlock(&fibril_futex);
|
---|
[514d561] | 674 | _fibril_cleanup_dead();
|
---|
| 675 | return rc;
|
---|
[bc1f1c2] | 676 | }
|
---|
| 677 |
|
---|
[514d561] | 678 | void fibril_wait_for(fibril_event_t *event)
|
---|
[bc1f1c2] | 679 | {
|
---|
[514d561] | 680 | (void) fibril_wait_timeout(event, NULL);
|
---|
| 681 | }
|
---|
[a35b458] | 682 |
|
---|
[514d561] | 683 | void fibril_notify(fibril_event_t *event)
|
---|
| 684 | {
|
---|
[df7cbc6] | 685 | futex_lock(&fibril_futex);
|
---|
[514d561] | 686 | _ready_list_push(_fibril_trigger_internal(event, _EVENT_TRIGGERED));
|
---|
[df7cbc6] | 687 | futex_unlock(&fibril_futex);
|
---|
[bc1f1c2] | 688 | }
|
---|
| 689 |
|
---|
[514d561] | 690 | /** Start a fibril that has not been running yet. */
|
---|
| 691 | void fibril_start(fibril_t *fibril)
|
---|
[bc1f1c2] | 692 | {
|
---|
[df7cbc6] | 693 | futex_lock(&fibril_futex);
|
---|
[514d561] | 694 | assert(!fibril->is_running);
|
---|
| 695 | fibril->is_running = true;
|
---|
| 696 |
|
---|
| 697 | if (!link_in_use(&fibril->all_link))
|
---|
| 698 | list_append(&fibril->all_link, &fibril_list);
|
---|
| 699 |
|
---|
| 700 | _ready_list_push(fibril);
|
---|
| 701 |
|
---|
[df7cbc6] | 702 | futex_unlock(&fibril_futex);
|
---|
[bc1f1c2] | 703 | }
|
---|
| 704 |
|
---|
[514d561] | 705 | /** Start a fibril that has not been running yet. (obsolete) */
|
---|
| 706 | void fibril_add_ready(fibril_t *fibril)
|
---|
| 707 | {
|
---|
| 708 | fibril_start(fibril);
|
---|
| 709 | }
|
---|
| 710 |
|
---|
| 711 | /** @return the currently running fibril. */
|
---|
[d73d992] | 712 | fibril_t *fibril_self(void)
|
---|
| 713 | {
|
---|
[40abf56] | 714 | assert(__tcb_is_set());
|
---|
| 715 | tcb_t *tcb = __tcb_get();
|
---|
| 716 | assert(tcb->fibril_data);
|
---|
| 717 | return tcb->fibril_data;
|
---|
[d73d992] | 718 | }
|
---|
| 719 |
|
---|
[514d561] | 720 | /**
|
---|
| 721 | * Obsolete, use fibril_self().
|
---|
[3562ec82] | 722 | *
|
---|
[514d561] | 723 | * @return ID of the currently running fibril.
|
---|
[bc1f1c2] | 724 | */
|
---|
| 725 | fid_t fibril_get_id(void)
|
---|
| 726 | {
|
---|
[d73d992] | 727 | return (fid_t) fibril_self();
|
---|
| 728 | }
|
---|
| 729 |
|
---|
[514d561] | 730 | /**
|
---|
| 731 | * Switch to another fibril, if one is ready to run.
|
---|
| 732 | * Has no effect on a heavy fibril.
|
---|
| 733 | */
|
---|
[d73d992] | 734 | void fibril_yield(void)
|
---|
| 735 | {
|
---|
[514d561] | 736 | fibril_t *f = _ready_list_pop_nonblocking(false);
|
---|
| 737 | if (f)
|
---|
| 738 | _fibril_switch_to(SWITCH_FROM_YIELD, f, false);
|
---|
[bc1f1c2] | 739 | }
|
---|
| 740 |
|
---|
[c124c985] | 741 | static void _runner_fn(void *arg)
|
---|
| 742 | {
|
---|
[514d561] | 743 | _helper_fibril_fn(arg);
|
---|
[c124c985] | 744 | }
|
---|
| 745 |
|
---|
| 746 | /**
|
---|
| 747 | * Spawn a given number of runners (i.e. OS threads) immediately, and
|
---|
| 748 | * unconditionally. This is meant to be used for tests and debugging.
|
---|
| 749 | * Regular programs should just use `fibril_enable_multithreaded()`.
|
---|
| 750 | *
|
---|
| 751 | * @param n Number of runners to spawn.
|
---|
| 752 | * @return Number of runners successfully spawned.
|
---|
| 753 | */
|
---|
| 754 | int fibril_test_spawn_runners(int n)
|
---|
| 755 | {
|
---|
[514d561] | 756 | if (!multithreaded)
|
---|
| 757 | multithreaded = true;
|
---|
| 758 |
|
---|
[c124c985] | 759 | errno_t rc;
|
---|
| 760 |
|
---|
| 761 | for (int i = 0; i < n; i++) {
|
---|
| 762 | thread_id_t tid;
|
---|
| 763 | rc = thread_create(_runner_fn, NULL, "fibril runner", &tid);
|
---|
| 764 | if (rc != EOK)
|
---|
| 765 | return i;
|
---|
| 766 | thread_detach(tid);
|
---|
| 767 | }
|
---|
| 768 |
|
---|
| 769 | return n;
|
---|
| 770 | }
|
---|
| 771 |
|
---|
| 772 | /**
|
---|
| 773 | * Opt-in to have more than one runner thread.
|
---|
| 774 | *
|
---|
| 775 | * Currently, a task only ever runs in one thread because multithreading
|
---|
| 776 | * might break some existing code.
|
---|
| 777 | *
|
---|
| 778 | * Eventually, the number of runner threads for a given task should become
|
---|
| 779 | * configurable in the environment and this function becomes no-op.
|
---|
| 780 | */
|
---|
| 781 | void fibril_enable_multithreaded(void)
|
---|
| 782 | {
|
---|
| 783 | // TODO: Implement better.
|
---|
| 784 | // For now, 4 total runners is a sensible default.
|
---|
[514d561] | 785 | if (!multithreaded) {
|
---|
| 786 | fibril_test_spawn_runners(3);
|
---|
| 787 | }
|
---|
[c124c985] | 788 | }
|
---|
| 789 |
|
---|
| 790 | /**
|
---|
| 791 | * Detach a fibril.
|
---|
| 792 | */
|
---|
| 793 | void fibril_detach(fid_t f)
|
---|
| 794 | {
|
---|
| 795 | // TODO: Currently all fibrils are detached by default, but they
|
---|
| 796 | // won't always be. Code that explicitly spawns fibrils with
|
---|
| 797 | // limited lifetime should call this function.
|
---|
| 798 | }
|
---|
| 799 |
|
---|
[514d561] | 800 | /**
|
---|
| 801 | * Exit a fibril. Never returns.
|
---|
| 802 | *
|
---|
| 803 | * @param retval Value to return from fibril_join() called on this fibril.
|
---|
| 804 | */
|
---|
| 805 | _Noreturn void fibril_exit(long retval)
|
---|
| 806 | {
|
---|
| 807 | // TODO: implement fibril_join() and remember retval
|
---|
| 808 | (void) retval;
|
---|
| 809 |
|
---|
| 810 | fibril_t *f = _ready_list_pop_nonblocking(false);
|
---|
| 811 | if (!f)
|
---|
| 812 | f = fibril_self()->thread_ctx;
|
---|
| 813 |
|
---|
| 814 | _fibril_switch_to(SWITCH_FROM_DEAD, f, false);
|
---|
| 815 | __builtin_unreachable();
|
---|
| 816 | }
|
---|
| 817 |
|
---|
| 818 | void __fibrils_init(void)
|
---|
| 819 | {
|
---|
| 820 | /*
|
---|
| 821 | * We allow a fixed, small amount of parallelism for IPC reads, but
|
---|
| 822 | * since IPC is currently serialized in kernel, there's not much
|
---|
| 823 | * we can get from more threads reading messages.
|
---|
| 824 | */
|
---|
| 825 |
|
---|
| 826 | #define IPC_BUFFER_COUNT 1024
|
---|
| 827 | static _ipc_buffer_t buffers[IPC_BUFFER_COUNT];
|
---|
| 828 |
|
---|
| 829 | for (int i = 0; i < IPC_BUFFER_COUNT; i++) {
|
---|
| 830 | list_append(&buffers[i].link, &ipc_buffer_free_list);
|
---|
| 831 | futex_up(&ready_semaphore);
|
---|
| 832 | }
|
---|
| 833 | }
|
---|
| 834 |
|
---|
| 835 | void fibril_usleep(suseconds_t timeout)
|
---|
| 836 | {
|
---|
| 837 | struct timeval expires;
|
---|
| 838 | getuptime(&expires);
|
---|
| 839 | tv_add_diff(&expires, timeout);
|
---|
| 840 |
|
---|
| 841 | fibril_event_t event = FIBRIL_EVENT_INIT;
|
---|
| 842 | fibril_wait_timeout(&event, &expires);
|
---|
| 843 | }
|
---|
| 844 |
|
---|
| 845 | void fibril_sleep(unsigned int sec)
|
---|
| 846 | {
|
---|
| 847 | struct timeval expires;
|
---|
| 848 | getuptime(&expires);
|
---|
| 849 | expires.tv_sec += sec;
|
---|
| 850 |
|
---|
| 851 | fibril_event_t event = FIBRIL_EVENT_INIT;
|
---|
| 852 | fibril_wait_timeout(&event, &expires);
|
---|
| 853 | }
|
---|
| 854 |
|
---|
| 855 | void fibril_ipc_poke(void)
|
---|
| 856 | {
|
---|
| 857 | DPRINTF("Poking.\n");
|
---|
| 858 | /* Wakeup one thread sleeping in SYS_IPC_WAIT. */
|
---|
| 859 | ipc_poke();
|
---|
| 860 | }
|
---|
| 861 |
|
---|
| 862 | errno_t fibril_ipc_wait(ipc_call_t *call, const struct timeval *expires)
|
---|
| 863 | {
|
---|
| 864 | return _wait_ipc(call, expires);
|
---|
| 865 | }
|
---|
| 866 |
|
---|
[bc1f1c2] | 867 | /** @}
|
---|
| 868 | */
|
---|