Index: kernel/test/synch/rcu1.c =================================================================== --- kernel/test/synch/rcu1.c (revision 1b20da07baaa3e3c424f62c927274e676e4295cd) +++ kernel/test/synch/rcu1.c (revision 12ae6d88e5835c9ddb7e8e846fddc0adcd611daa) @@ -67,5 +67,5 @@ TPRINTF("."); } - + if (!p->exited) { *presult = ETIMEOUT; @@ -81,8 +81,8 @@ { assert(thread[k] == NULL); - + thread[k] = thread_create(func, arg, TASK, THREAD_FLAG_NONE, "test-rcu-thread"); - + if(thread[k]) { /* Distribute evenly. */ @@ -95,7 +95,7 @@ { size_t thread_cnt = get_thread_cnt(); - + one_idx = 0; - + for (size_t i = 0; i < thread_cnt; ++i) { run_thread(i, func, NULL); @@ -106,7 +106,7 @@ { size_t thread_cnt = get_thread_cnt(); - + one_idx = 0; - + for (size_t i = 0; i < thread_cnt; ++i) { if (thread[i]) { @@ -115,10 +115,10 @@ errno_t ret = thread_join_timeout(thread[i], 5 * 1000 * 1000, 0); joined = (ret != ETIMEOUT); - + if (ret == EOK) { TPRINTF("%zu threads remain\n", thread_cnt - i - 1); } } while (!joined); - + thread_detach(thread[i]); thread[i] = NULL; @@ -140,5 +140,5 @@ --one_idx; - + if (thread[one_idx]) { thread_join(thread[one_idx]); @@ -154,12 +154,12 @@ { size_t nop_iters = (size_t)arg; - + TPRINTF("Enter nop-reader\n"); - + for (size_t i = 0; i < nop_iters; ++i) { rcu_read_lock(); rcu_read_unlock(); } - + TPRINTF("Exit nop-reader\n"); } @@ -169,9 +169,9 @@ assert(0 < steps && from <= to && 0 < to); size_t inc = (to - from) / (steps - 1); - + for (size_t i = 0; i < steps - 1; ++i) { seq[i] = i * inc + from; } - + seq[steps - 1] = to; } @@ -181,13 +181,13 @@ size_t seq[MAX_THREADS] = {0}; get_seq(100, 100000, get_thread_cnt(), seq); - + TPRINTF("\nRun %zu thr: repeat empty no-op reader sections\n", get_thread_cnt()); - + for (size_t k = 0; k < get_thread_cnt(); ++k) run_one(nop_reader, (void*)seq[k]); - + TPRINTF("\nJoining %zu no-op readers\n", get_thread_cnt()); join_all(); - + return true; } @@ -202,17 +202,17 @@ size_t nop_iters = (size_t)arg; size_t outer_iters = iter_cnt / nop_iters; - + TPRINTF("Enter long-reader\n"); - + for (size_t i = 0; i < outer_iters; ++i) { rcu_read_lock(); - + for (volatile size_t k = 0; k < nop_iters; ++k) { /* nop, but increment volatile k */ } - + rcu_read_unlock(); } - + TPRINTF("Exit long-reader\n"); } @@ -222,14 +222,14 @@ size_t seq[MAX_THREADS] = {0}; get_seq(10, 1000 * 1000, get_thread_cnt(), seq); - + TPRINTF("\nRun %zu thr: repeat long reader sections, will preempt, no cbs.\n", get_thread_cnt()); - + for (size_t k = 0; k < get_thread_cnt(); ++k) run_one(long_reader, (void*)seq[k]); - + TPRINTF("\nJoining %zu readers with long reader sections.\n", get_thread_cnt()); join_all(); - + return true; } @@ -253,5 +253,5 @@ rcu_item_t *a = malloc(sizeof(rcu_item_t), FRAME_ATOMIC); rcu_item_t *b = malloc(sizeof(rcu_item_t), FRAME_ATOMIC); - + if (a && b) { rcu_call(a, count_cb); @@ -272,12 +272,12 @@ size_t exp_cnt = nop_updater_iters * get_thread_cnt(); size_t max_used_mem = sizeof(rcu_item_t) * exp_cnt; - + TPRINTF("\nRun %zu thr: post %zu no-op callbacks (%zu B used), no readers.\n", get_thread_cnt(), exp_cnt, max_used_mem); - + run_all(nop_updater); TPRINTF("\nJoining %zu no-op callback threads\n", get_thread_cnt()); join_all(); - + size_t loop_cnt = 0, max_loops = 15; @@ -287,5 +287,5 @@ thread_sleep(1); } - + return loop_cnt < max_loops; } @@ -312,9 +312,9 @@ { TPRINTF("Enter one-cb-reader\n"); - - rcu_read_lock(); - + + rcu_read_lock(); + item_w_cookie_t *item = malloc(sizeof(item_w_cookie_t), FRAME_ATOMIC); - + if (item) { item->cookie = magic_cookie; @@ -323,9 +323,9 @@ TPRINTF("\n[out-of-mem]\n"); } - + thread_sleep(1); - + rcu_read_unlock(); - + TPRINTF("Exit one-cb-reader\n"); } @@ -334,18 +334,18 @@ { one_cb_is_done = 0; - + TPRINTF("\nRun a single reader that posts one callback.\n"); run_one(one_cb_reader, NULL); join_one(); - + TPRINTF("\nJoined one-cb reader, wait for callback.\n"); size_t loop_cnt = 0; size_t max_loops = 4; /* 200 ms total */ - + while (!one_cb_is_done && loop_cnt < max_loops) { thread_usleep(50 * 1000); ++loop_cnt; } - + return one_cb_is_done; } @@ -373,9 +373,9 @@ { seq_item_t *item = member_to_inst(rcu_item, seq_item_t, rcu); - + /* Racy but errs to the conservative side, so it is ok. */ if (max_upd_done_time < item->start_time) { max_upd_done_time = item->start_time; - + /* Make updated time visible */ memory_barrier(); @@ -393,5 +393,5 @@ #ifndef KARCH_riscv64 seq_work_t *work = (seq_work_t*)arg; - + /* Alternate between reader and updater roles. */ for (size_t k = 0; k < work->iters; ++k) { @@ -400,31 +400,31 @@ rcu_read_lock(); atomic_count_t start_time = atomic_postinc(&cur_time); - + for (volatile size_t d = 0; d < 10 * i; ++d ){ /* no-op */ } - + /* Get most recent max_upd_done_time. */ memory_barrier(); - + if (start_time < max_upd_done_time) { seq_test_result = ERACE; } - + rcu_read_unlock(); - + if (seq_test_result != EOK) return; } - + /* Updater */ for (size_t i = 0; i < work->update_cnt; ++i) { seq_item_t *a = malloc(sizeof(seq_item_t), FRAME_ATOMIC); seq_item_t *b = malloc(sizeof(seq_item_t), FRAME_ATOMIC); - + if (a && b) { a->start_time = atomic_postinc(&cur_time); rcu_call(&a->rcu, seq_cb); - + b->start_time = atomic_postinc(&cur_time); rcu_call(&b->rcu, seq_cb); @@ -437,5 +437,5 @@ } } - + } #else @@ -454,10 +454,10 @@ size_t read_cnt[MAX_THREADS] = {0}; seq_work_t item[MAX_THREADS]; - + size_t total_cbs = 0; size_t max_used_mem = 0; - + get_seq(0, total_cnt, get_thread_cnt(), read_cnt); - + for (size_t i = 0; i < get_thread_cnt(); ++i) { @@ -465,8 +465,8 @@ item[i].read_cnt = read_cnt[i]; item[i].iters = iters; - + total_cbs += 2 * iters * item[i].update_cnt; } - + max_used_mem = total_cbs * sizeof(seq_item_t); @@ -474,16 +474,16 @@ uint64_t mem_units; bin_order_suffix(max_used_mem, &mem_units, &mem_suffix, false); - + TPRINTF("\nRun %zu th: check callback completion time in readers. " "%zu callbacks total (max %" PRIu64 " %s used). Be patient.\n", get_thread_cnt(), total_cbs, mem_units, mem_suffix); - + for (size_t i = 0; i < get_thread_cnt(); ++i) { run_one(seq_func, &item[i]); } - + TPRINTF("\nJoining %zu seq-threads\n", get_thread_cnt()); join_all(); - + if (seq_test_result == ENOMEM) { TPRINTF("\nErr: out-of mem\n"); @@ -491,5 +491,5 @@ TPRINTF("\nERROR: race detected!!\n"); } - + return seq_test_result == EOK; } @@ -510,10 +510,10 @@ rcu_read_lock(); rcu_read_unlock(); - + rcu_call((rcu_item_t*)arg, reader_unlocked); - - rcu_read_lock(); - rcu_read_lock(); - + + rcu_read_lock(); + rcu_read_lock(); + /* Exit without unlocking the rcu reader section. */ } @@ -522,5 +522,5 @@ { TPRINTF("\nReader exits thread with rcu_lock\n"); - + exited_t *p = malloc(sizeof(exited_t), FRAME_ATOMIC); if (!p) { @@ -528,13 +528,13 @@ return false; } - + p->exited = false; - + run_one(reader_exit, p); join_one(); - + errno_t result = EOK; wait_for_cb_exit(2 /* secs */, p, &result); - + if (result != EOK) { TPRINTF("Err: RCU locked up after exiting from within a reader\n"); @@ -543,5 +543,5 @@ free(p); } - + return result == EOK; } @@ -570,5 +570,5 @@ TPRINTF("reader_prev{ "); - + rcu_read_lock(); scheduler(); @@ -588,5 +588,5 @@ preempt_t *p = (preempt_t*)arg; assert(!p->e.exited); - + TPRINTF("reader_inside_cur{ "); /* @@ -613,5 +613,5 @@ preempt_t *p = (preempt_t*)arg; assert(!p->e.exited); - + TPRINTF("reader_cur{ "); rcu_read_lock(); @@ -622,5 +622,5 @@ /* Preempt while cur GP detection is running */ thread_sleep(1); - + /* Err: exited before this reader completed. */ if (p->e.exited) @@ -635,5 +635,5 @@ preempt_t *p = (preempt_t*)arg; assert(!p->e.exited); - + TPRINTF("reader_next1{ "); rcu_read_lock(); @@ -641,5 +641,5 @@ /* Preempt before cur GP detection starts. */ scheduler(); - + /* Start GP. */ rcu_call(&p->e.rcu, preempted_unlocked); @@ -657,5 +657,5 @@ preempt_t *p = (preempt_t*)arg; assert(!p->e.exited); - + TPRINTF("reader_next2{ "); rcu_read_lock(); @@ -663,5 +663,5 @@ /* Preempt before cur GP detection starts. */ scheduler(); - + /* Start GP. */ rcu_call(&p->e.rcu, preempted_unlocked); @@ -691,14 +691,14 @@ return false; } - + p->e.exited = false; p->result = EOK; - + run_one(f, p); join_one(); - + /* Wait at most 4 secs. */ wait_for_cb_exit(4, &p->e, &p->result); - + if (p->result == EOK) { free(p); @@ -714,20 +714,20 @@ { TPRINTF("\nReaders will be preempted.\n"); - + bool success = true; bool ok = true; - + ok = do_one_reader_preempt(preempted_reader_prev, "Err: preempted_reader_prev()\n"); success = success && ok; - + ok = do_one_reader_preempt(preempted_reader_inside_cur, "Err: preempted_reader_inside_cur()\n"); success = success && ok; - + ok = do_one_reader_preempt(preempted_reader_cur, "Err: preempted_reader_cur()\n"); success = success && ok; - + ok = do_one_reader_preempt(preempted_reader_next1, "Err: preempted_reader_next1()\n"); @@ -737,5 +737,5 @@ "Err: preempted_reader_next2()\n"); success = success && ok; - + return success; } @@ -751,22 +751,22 @@ { synch_t *synch = (synch_t *) arg; - + rcu_read_lock(); /* Order accesses of synch after the reader section begins. */ memory_barrier(); - + synch->reader_running = true; - + while (!synch->synch_running) { /* 0.5 sec */ delay(500 * 1000); } - + /* Run for 1 sec */ delay(1000 * 1000); /* thread_join() propagates done to do_synch() */ synch->reader_done = true; - + rcu_read_unlock(); } @@ -776,18 +776,18 @@ { TPRINTF("\nSynchronize with long reader\n"); - + synch_t *synch = malloc(sizeof(synch_t), FRAME_ATOMIC); - + if (!synch) { TPRINTF("[out-of-mem]\n"); return false; } - + synch->reader_done = false; synch->reader_running = false; synch->synch_running = false; - + run_one(synch_reader, synch); - + /* Wait for the reader to enter its critical section. */ scheduler(); @@ -795,11 +795,11 @@ thread_usleep(500 * 1000); } - + synch->synch_running = true; - + rcu_synchronize(); join_one(); - - + + if (synch->reader_done) { free(synch); @@ -827,17 +827,17 @@ { TPRINTF("\nrcu_barrier: Wait for outstanding rcu callbacks to complete\n"); - + barrier_t *barrier = malloc(sizeof(barrier_t), FRAME_ATOMIC); - + if (!barrier) { TPRINTF("[out-of-mem]\n"); return false; } - + atomic_set(&barrier->done, 0); - + rcu_call(&barrier->rcu_item, barrier_callback); rcu_barrier(); - + if (1 == atomic_get(&barrier->done)) { free(barrier); @@ -861,9 +861,9 @@ { bool *done = (bool*) arg; - + while (!*done) { rcu_read_lock(); rcu_read_unlock(); - + /* * Do some work outside of the reader section so we are not always @@ -884,8 +884,8 @@ { stress_t *s = (stress_t *)arg; - + for (size_t i = 0; i < s->iters; ++i) { rcu_item_t *item = malloc(sizeof(rcu_item_t), FRAME_ATOMIC); - + if (item) { rcu_call(item, stress_cb); @@ -894,5 +894,5 @@ return; } - + /* Print a dot if we make a progress of 1% */ if (s->master && 0 == (i % (s->iters/100))) @@ -907,13 +907,13 @@ stress_t master = { .iters = cb_per_thread, .master = true }; stress_t worker = { .iters = cb_per_thread, .master = false }; - + size_t thread_cnt = min(MAX_THREADS / 2, config.cpu_active); /* Each cpu has one reader and one updater. */ size_t reader_cnt = thread_cnt; size_t updater_cnt = thread_cnt; - + size_t exp_upd_calls = updater_cnt * cb_per_thread; size_t max_used_mem = exp_upd_calls * sizeof(rcu_item_t); - + const char *mem_suffix; uint64_t mem_units; @@ -923,5 +923,5 @@ " total (max %" PRIu64 " %s used). Be very patient.\n", reader_cnt, updater_cnt, exp_upd_calls, mem_units, mem_suffix); - + for (size_t k = 0; k < reader_cnt; ++k) { run_one(stress_reader, &done); @@ -931,15 +931,15 @@ run_one(stress_updater, k > 0 ? &worker : &master); } - + TPRINTF("\nJoining %zu stress updaters.\n", updater_cnt); - + for (size_t k = 0; k < updater_cnt; ++k) { join_one(); } - + done = true; TPRINTF("\nJoining %zu stress nop-readers.\n", reader_cnt); - + join_all(); return true; @@ -957,12 +957,12 @@ { expedite_t *e = (expedite_t *)arg; - + if (1 < e->count_down) { --e->count_down; - + if (0 == (e->count_down % (e->total_cnt/100))) { TPRINTF("*"); } - + _rcu_call(e->expedite, &e->r, expedite_cb); } else { @@ -979,7 +979,7 @@ e.count_down = cnt; e.expedite = exp; - + _rcu_call(e.expedite, &e.r, expedite_cb); - + while (0 < e.count_down) { thread_sleep(1); @@ -992,5 +992,5 @@ size_t exp_cnt = 1000 * 1000; size_t normal_cnt = 1 * 1000; - + TPRINTF("Expedited: sequence of %zu rcu_calls\n", exp_cnt); run_expedite(true, exp_cnt); @@ -1024,10 +1024,10 @@ { 0, NULL, NULL } }; - + bool success = true; bool ok = true; uint64_t completed_gps = rcu_completed_gps(); uint64_t delta_gps = 0; - + for (int i = 0; test_func[i].func; ++i) { if (!test_func[i].include) { @@ -1037,8 +1037,8 @@ TPRINTF("\nRunning subtest %s.\n", test_func[i].desc); } - + ok = test_func[i].func(); success = success && ok; - + delta_gps = rcu_completed_gps() - completed_gps; completed_gps += delta_gps; Index: kernel/test/synch/semaphore1.c =================================================================== --- kernel/test/synch/semaphore1.c (revision 1b20da07baaa3e3c424f62c927274e676e4295cd) +++ kernel/test/synch/semaphore1.c (revision 12ae6d88e5835c9ddb7e8e846fddc0adcd611daa) @@ -49,7 +49,7 @@ { thread_detach(THREAD); - + waitq_sleep(&can_start); - + semaphore_down(&sem); atomic_inc(&items_produced); @@ -61,7 +61,7 @@ { thread_detach(THREAD); - + waitq_sleep(&can_start); - + semaphore_down(&sem); atomic_inc(&items_consumed); @@ -75,20 +75,20 @@ atomic_count_t consumers; atomic_count_t producers; - + waitq_initialize(&can_start); semaphore_initialize(&sem, AT_ONCE); - + for (i = 1; i <= 3; i++) { thread_t *thrd; - + atomic_set(&items_produced, 0); atomic_set(&items_consumed, 0); - + consumers = i * CONSUMERS; producers = (4 - i) * PRODUCERS; - + TPRINTF("Creating %" PRIua " consumers and %" PRIua " producers...", consumers, producers); - + for (j = 0; j < (CONSUMERS + PRODUCERS) / 2; j++) { for (k = 0; k < i; k++) { @@ -109,10 +109,10 @@ } } - + TPRINTF("ok\n"); - + thread_sleep(1); waitq_wakeup(&can_start, WAKEUP_ALL); - + while ((items_consumed.count != consumers) || (items_produced.count != producers)) { TPRINTF("%" PRIua " consumers remaining, %" PRIua " producers remaining\n", @@ -121,5 +121,5 @@ } } - + return NULL; } Index: kernel/test/synch/semaphore2.c =================================================================== --- kernel/test/synch/semaphore2.c (revision 1b20da07baaa3e3c424f62c927274e676e4295cd) +++ kernel/test/synch/semaphore2.c (revision 12ae6d88e5835c9ddb7e8e846fddc0adcd611daa) @@ -50,5 +50,5 @@ { uint32_t rc; - + spinlock_lock(&sem_lock); rc = seed % max; @@ -62,9 +62,9 @@ errno_t rc; int to; - + thread_detach(THREAD); - + waitq_sleep(&can_start); - + to = random(20000); TPRINTF("cpu%u, tid %" PRIu64 " down+ (%d)\n", CPU->id, THREAD->tid, to); @@ -74,8 +74,8 @@ return; } - + TPRINTF("cpu%u, tid %" PRIu64 " down=\n", CPU->id, THREAD->tid); thread_usleep(random(30000)); - + semaphore_up(&sem); TPRINTF("cpu%u, tid %" PRIu64 " up\n", CPU->id, THREAD->tid); @@ -85,10 +85,10 @@ { uint32_t i, k; - + waitq_initialize(&can_start); semaphore_initialize(&sem, 5); - + thread_t *thrd; - + k = random(7) + 1; TPRINTF("Creating %" PRIu32 " consumers\n", k); @@ -101,8 +101,8 @@ TPRINTF("Error creating thread\n"); } - + thread_usleep(20000); waitq_wakeup(&can_start, WAKEUP_ALL); - + return NULL; } Index: kernel/test/synch/workq-test-core.h =================================================================== --- kernel/test/synch/workq-test-core.h (revision 1b20da07baaa3e3c424f62c927274e676e4295cd) +++ kernel/test/synch/workq-test-core.h (revision 12ae6d88e5835c9ddb7e8e846fddc0adcd611daa) @@ -54,5 +54,5 @@ { ++work->wave; - + if (work->wave < WAVES) { work->count_down = COUNT; @@ -79,5 +79,5 @@ child->count_down = work->count_down; } - + return child; } @@ -93,9 +93,9 @@ /* Ensure work_item is ours for the taking. */ memsetb(work_item, sizeof(work_t), 0xec); - + test_work_t *work = (test_work_t *)work_item; - + atomic_inc(&call_cnt[work->wave]); - + if (0 < work->count_down) { /* Sleep right before creating the last generation. */ @@ -108,5 +108,5 @@ } } - + --work->count_down; @@ -122,5 +122,5 @@ } } - + if (!core_workq_enqueue(work_item, reproduce)) { if (work->master) @@ -131,5 +131,5 @@ } else { /* We're done with this wave - only the master survives. */ - + if (work->master && new_wave(work)) { if (!core_workq_enqueue(work_item, reproduce)) { @@ -140,5 +140,5 @@ if (work->master) TPRINTF("\nMaster work item done.\n"); - + free_work(work); } @@ -157,5 +157,5 @@ work->wave = 0; work->count_down = COUNT; - + /* * k == COUNT_POW @@ -166,16 +166,16 @@ */ size_t exp_call_cnt = (COUNT_POW + 2) * (1 << (COUNT_POW - 1)); - + TPRINTF("waves: %d, count_down: %d, total expected calls: %zu\n", WAVES, COUNT, exp_call_cnt * WAVES); - + core_workq_enqueue(&work->work_item, reproduce); - + size_t sleep_cnt = 0; /* At least 40 seconds total (or 2 sec to end while there's work). */ size_t max_sleep_secs = end_prematurely ? 2 : MAIN_MAX_SLEEP_SEC; size_t max_sleep_cnt = (max_sleep_secs * 1000) / MAIN_POLL_SLEEP_MS; - + for (int i = 0; i < WAVES; ++i) { while (atomic_get(&call_cnt[i]) < exp_call_cnt @@ -186,7 +186,7 @@ } } - + bool success = true; - + for (int i = 0; i < WAVES; ++i) { if (atomic_get(&call_cnt[i]) == exp_call_cnt) { @@ -199,6 +199,6 @@ } } - - + + if (success) return NULL; Index: kernel/test/synch/workqueue2.c =================================================================== --- kernel/test/synch/workqueue2.c (revision 1b20da07baaa3e3c424f62c927274e676e4295cd) +++ kernel/test/synch/workqueue2.c (revision 12ae6d88e5835c9ddb7e8e846fddc0adcd611daa) @@ -63,5 +63,5 @@ basic_done = 0; workq_global_enqueue(&basic_work, basic_test_work); - + while (!basic_done) { TPRINTF("."); @@ -87,14 +87,14 @@ { workq = workq_create(qname); - + if (!workq) { return "Failed to create a work queue.\n"; } - + const char *ret = run_workq_core(stop); - + TPRINTF("Stopping work queue...\n"); workq_stop(workq); - + TPRINTF("Destroying work queue...\n"); workq_destroy(workq); @@ -123,7 +123,7 @@ const char *err = NULL; const char *res; - + basic_test(); - + res = test_custom_workq(); if (res) { @@ -131,5 +131,5 @@ err = res; } - + res = test_custom_workq_stop(); if (res) { @@ -137,5 +137,5 @@ err = res; } - + res = test_workqueue3(); if (res) { Index: kernel/test/synch/workqueue3.c =================================================================== --- kernel/test/synch/workqueue3.c (revision 1b20da07baaa3e3c424f62c927274e676e4295cd) +++ kernel/test/synch/workqueue3.c (revision 12ae6d88e5835c9ddb7e8e846fddc0adcd611daa) @@ -66,5 +66,5 @@ TPRINTF("Done.\n"); - + return err; }