Changes in kernel/generic/src/proc/thread.c [dfa4be62:b169619] in mainline

File:

• kernel/generic/src/proc/thread.c (modified) (28 diffs)

kernel/generic/src/proc/thread.c

@@ -60 +60 @@
 #include <arch/interrupt.h>
 #include <smp/ipi.h>
+#include <arch/faddr.h>
 #include <atomic.h>
 #include <memw.h>
@@ -81 +82 @@
 };
 
+enum sleep_state {
+        SLEEP_INITIAL,
+        SLEEP_ASLEEP,
+        SLEEP_WOKE,
+};
+
 /** Lock protecting the @c threads ordered dictionary .
  *
@@ -108 +115 @@
 static int threads_cmp(void *, void *);
 
+/** Thread wrapper.
+ *
+ * This wrapper is provided to ensure that every thread makes a call to
+ * thread_exit() when its implementing function returns.
+ *
+ * interrupts_disable() is assumed.
+ *
+ */
+static void cushion(void)
+{
+        void (*f)(void *) = THREAD->thread_code;
+        void *arg = THREAD->thread_arg;
+        THREAD->last_cycle = get_cycle();
+
+        /* This is where each thread wakes up after its creation */
+        irq_spinlock_unlock(&THREAD->lock, false);
+        interrupts_enable();
+
+        f(arg);
+
+        thread_exit();
+
+        /* Not reached */
+}
+
 /** Initialization and allocation for thread_t structure
  *
@@ -115 +147 @@
         thread_t *thread = (thread_t *) obj;
 
+        irq_spinlock_initialize(&thread->lock, "thread_t_lock");
         link_initialize(&thread->rq_link);
         link_initialize(&thread->wq_link);
@@ -196 +229 @@
 void thread_wire(thread_t *thread, cpu_t *cpu)
 {
-        ipl_t ipl = interrupts_disable();
-        atomic_set_unordered(&thread->cpu, cpu);
+        irq_spinlock_lock(&thread->lock, true);
+        thread->cpu = cpu;
         thread->nomigrate++;
-        interrupts_restore(ipl);
-}
-
-/** Start a thread that wasn't started yet since it was created.
- *
- * @param thread A reference to the newly created thread.
- */
-void thread_start(thread_t *thread)
-{
-        assert(atomic_get_unordered(&thread->state) == Entering);
-        thread_requeue_sleeping(thread_ref(thread));
+        irq_spinlock_unlock(&thread->lock, true);
+}
+
+/** Invoked right before thread_ready() readies the thread. thread is locked. */
+static void before_thread_is_ready(thread_t *thread)
+{
+        assert(irq_spinlock_locked(&thread->lock));
+}
+
+/** Make thread ready
+ *
+ * Switch thread to the ready state. Consumes reference passed by the caller.
+ *
+ * @param thread Thread to make ready.
+ *
+ */
+void thread_ready(thread_t *thread)
+{
+        irq_spinlock_lock(&thread->lock, true);
+
+        assert(thread->state != Ready);
+
+        before_thread_is_ready(thread);
+
+        int i = (thread->priority < RQ_COUNT - 1) ?
+            ++thread->priority : thread->priority;
+
+        /* Prefer the CPU on which the thread ran last */
+        cpu_t *cpu = thread->cpu ? thread->cpu : CPU;
+
+        thread->state = Ready;
+
+        irq_spinlock_pass(&thread->lock, &(cpu->rq[i].lock));
+
+        /*
+         * Append thread to respective ready queue
+         * on respective processor.
+         */
+
+        list_append(&thread->rq_link, &cpu->rq[i].rq);
+        cpu->rq[i].n++;
+        irq_spinlock_unlock(&(cpu->rq[i].lock), true);
+
+        atomic_inc(&nrdy);
+        atomic_inc(&cpu->nrdy);
 }
 
@@ -248 +315 @@
         irq_spinlock_unlock(&tidlock, true);
 
-        context_create(&thread->saved_context, thread_main_func,
-            thread->kstack, STACK_SIZE);
+        memset(&thread->saved_context, 0, sizeof(thread->saved_context));
+        context_set(&thread->saved_context, FADDR(cushion),
+            (uintptr_t) thread->kstack, STACK_SIZE);
 
         current_initialize((current_t *) thread->kstack);
+
+        ipl_t ipl = interrupts_disable();
+        thread->saved_ipl = interrupts_read();
+        interrupts_restore(ipl);
 
         str_cpy(thread->name, THREAD_NAME_BUFLEN, name);
@@ -257 +329 @@
         thread->thread_code = func;
         thread->thread_arg = arg;
-        thread->ucycles = ATOMIC_TIME_INITIALIZER();
-        thread->kcycles = ATOMIC_TIME_INITIALIZER();
+        thread->ucycles = 0;
+        thread->kcycles = 0;
         thread->uncounted =
             ((flags & THREAD_FLAG_UNCOUNTED) == THREAD_FLAG_UNCOUNTED);
-        atomic_init(&thread->priority, 0);
-        atomic_init(&thread->cpu, NULL);
+        thread->priority = -1;          /* Start in rq[0] */
+        thread->cpu = NULL;
         thread->stolen = false;
         thread->uspace =
@@ -268 +340 @@
 
         thread->nomigrate = 0;
-        atomic_init(&thread->state, Entering);
+        thread->state = Entering;
 
         atomic_init(&thread->sleep_queue, NULL);
@@ -288 +360 @@
 #ifdef CONFIG_UDEBUG
         /* Initialize debugging stuff */
-        atomic_init(&thread->btrace, false);
+        thread->btrace = false;
         udebug_thread_initialize(&thread->udebug);
 #endif
@@ -332 +404 @@
 
         if (!thread->uncounted) {
-                thread->task->ucycles += atomic_time_read(&thread->ucycles);
-                thread->task->kcycles += atomic_time_read(&thread->kcycles);
+                thread->task->ucycles += thread->ucycles;
+                thread->task->kcycles += thread->kcycles;
         }
 
         irq_spinlock_unlock(&thread->task->lock, false);
 
-        assert((atomic_get_unordered(&thread->state) == Exiting) || (atomic_get_unordered(&thread->state) == Lingering));
+        assert((thread->state == Exiting) || (thread->state == Lingering));
 
         /* Clear cpu->fpu_owner if set to this thread. */
 #ifdef CONFIG_FPU_LAZY
-        cpu_t *cpu = atomic_get_unordered(&thread->cpu);
-        if (cpu) {
+        if (thread->cpu) {
                 /*
                  * We need to lock for this because the old CPU can concurrently try
@@ -349 +420 @@
                  * it to finish. An atomic compare-and-swap wouldn't be enough.
                  */
-                irq_spinlock_lock(&cpu->fpu_lock, false);
-
-                if (atomic_get_unordered(&cpu->fpu_owner) == thread)
-                        atomic_set_unordered(&cpu->fpu_owner, NULL);
-
-                irq_spinlock_unlock(&cpu->fpu_lock, false);
+                irq_spinlock_lock(&thread->cpu->fpu_lock, false);
+
+                thread_t *owner = atomic_load_explicit(&thread->cpu->fpu_owner,
+                    memory_order_relaxed);
+
+                if (owner == thread) {
+                        atomic_store_explicit(&thread->cpu->fpu_owner, NULL,
+                            memory_order_relaxed);
+                }
+
+                irq_spinlock_unlock(&thread->cpu->fpu_lock, false);
         }
 #endif
@@ -449 +525 @@
         }
 
-        scheduler_enter(Exiting);
-        unreachable();
+        irq_spinlock_lock(&THREAD->lock, true);
+        THREAD->state = Exiting;
+        irq_spinlock_unlock(&THREAD->lock, true);
+
+        scheduler();
+
+        panic("should never be reached");
 }
 
@@ -496 +577 @@
 
         return THREAD->interrupted ? THREAD_TERMINATING : THREAD_OK;
+}
+
+static void thread_wait_internal(void)
+{
+        assert(THREAD != NULL);
+
+        ipl_t ipl = interrupts_disable();
+
+        if (atomic_load(&haltstate))
+                halt();
+
+        /*
+         * Lock here to prevent a race between entering the scheduler and another
+         * thread rescheduling this thread.
+         */
+        irq_spinlock_lock(&THREAD->lock, false);
+
+        int expected = SLEEP_INITIAL;
+
+        /* Only set SLEEP_ASLEEP in sleep pad if it's still in initial state */
+        if (atomic_compare_exchange_strong_explicit(&THREAD->sleep_state, &expected,
+            SLEEP_ASLEEP, memory_order_acq_rel, memory_order_acquire)) {
+                THREAD->state = Sleeping;
+                scheduler_locked(ipl);
+        } else {
+                assert(expected == SLEEP_WOKE);
+                /* Return immediately. */
+                irq_spinlock_unlock(&THREAD->lock, false);
+                interrupts_restore(ipl);
+        }
 }
 
@@ -538 +649 @@
         timeout_t timeout;
 
-        /* Extra check to avoid going to scheduler if we don't need to. */
-        if (atomic_load_explicit(&THREAD->sleep_state, memory_order_acquire) !=
-            SLEEP_INITIAL)
-                return THREAD_WAIT_SUCCESS;
-
         if (deadline != DEADLINE_NEVER) {
+                /* Extra check to avoid setting up a deadline if we don't need to. */
+                if (atomic_load_explicit(&THREAD->sleep_state, memory_order_acquire) !=
+                    SLEEP_INITIAL)
+                        return THREAD_WAIT_SUCCESS;
+
                 timeout_initialize(&timeout);
                 timeout_register_deadline(&timeout, deadline,
@@ -549 +660 @@
         }
 
-        scheduler_enter(Sleeping);
+        thread_wait_internal();
 
         if (deadline != DEADLINE_NEVER && !timeout_unregister(&timeout)) {
@@ -563 +674 @@
 
         int state = atomic_exchange_explicit(&thread->sleep_state, SLEEP_WOKE,
-            memory_order_acq_rel);
+            memory_order_release);
 
         if (state == SLEEP_ASLEEP) {
@@ -571 +682 @@
                  * the waking thread by the sleeper in thread_wait_finish().
                  */
-                thread_requeue_sleeping(thread);
+                thread_ready(thread);
         }
 }
@@ -578 +689 @@
 void thread_migration_disable(void)
 {
-        ipl_t ipl = interrupts_disable();
-
         assert(THREAD);
+
         THREAD->nomigrate++;
-
-        interrupts_restore(ipl);
 }
 
@@ -589 +697 @@
 void thread_migration_enable(void)
 {
-        ipl_t ipl = interrupts_disable();
-
         assert(THREAD);
         assert(THREAD->nomigrate > 0);
@@ -596 +702 @@
         if (THREAD->nomigrate > 0)
                 THREAD->nomigrate--;
-
-        interrupts_restore(ipl);
 }
 
@@ -627 +731 @@
 
 /** Wait for another thread to exit.
- * After successful wait, the thread reference is destroyed.
+ * This function does not destroy the thread. Reference counting handles that.
  *
  * @param thread Thread to join on exit.
@@ -638 +742 @@
 errno_t thread_join_timeout(thread_t *thread, uint32_t usec, unsigned int flags)
 {
-        assert(thread != NULL);
-
         if (thread == THREAD)
                 return EINVAL;
 
-        errno_t rc = _waitq_sleep_timeout(&thread->join_wq, usec, flags);
-
-        if (rc == EOK)
-                thread_put(thread);
-
-        return rc;
-}
-
-void thread_detach(thread_t *thread)
-{
-        thread_put(thread);
+        irq_spinlock_lock(&thread->lock, true);
+        state_t state = thread->state;
+        irq_spinlock_unlock(&thread->lock, true);
+
+        if (state == Exiting) {
+                return EOK;
+        } else {
+                return _waitq_sleep_timeout(&thread->join_wq, usec, flags);
+        }
 }
 
@@ -670 +770 @@
 
         (void) waitq_sleep_timeout(&wq, usec);
-}
-
-/** Allow other threads to run. */
-void thread_yield(void)
-{
-        assert(THREAD != NULL);
-        scheduler_enter(Running);
 }
 
@@ -683 +776 @@
         uint64_t ucycles, kcycles;
         char usuffix, ksuffix;
-        order_suffix(atomic_time_read(&thread->ucycles), &ucycles, &usuffix);
-        order_suffix(atomic_time_read(&thread->kcycles), &kcycles, &ksuffix);
-
-        state_t state = atomic_get_unordered(&thread->state);
+        order_suffix(thread->ucycles, &ucycles, &usuffix);
+        order_suffix(thread->kcycles, &kcycles, &ksuffix);
 
         char *name;
@@ -700 +791 @@
         else
                 printf("%-8" PRIu64 " %-14s %p %-8s %p %-5" PRIu32 "\n",
-                    thread->tid, name, thread, thread_states[state],
+                    thread->tid, name, thread, thread_states[thread->state],
                     thread->task, thread->task->container);
 
         if (additional) {
-                cpu_t *cpu = atomic_get_unordered(&thread->cpu);
-                if (cpu)
-                        printf("%-5u", cpu->id);
+                if (thread->cpu)
+                        printf("%-5u", thread->cpu->id);
                 else
                         printf("none ");
 
-                if (state == Sleeping) {
+                if (thread->state == Sleeping) {
                         printf(" %p", thread->sleep_queue);
                 }
@@ -789 +879 @@
 void thread_update_accounting(bool user)
 {
+        uint64_t time = get_cycle();
+
         assert(interrupts_disabled());
-
-        uint64_t time = get_cycle();
+        assert(irq_spinlock_locked(&THREAD->lock));
 
         if (user)
-                atomic_time_increment(&THREAD->ucycles, time - THREAD->last_cycle);
+                THREAD->ucycles += time - THREAD->last_cycle;
         else
-                atomic_time_increment(&THREAD->kcycles, time - THREAD->last_cycle);
+                THREAD->kcycles += time - THREAD->last_cycle;
 
         THREAD->last_cycle = time;
@@ -907 +998 @@
          */
 
-        printf("Scheduling thread stack trace.\n");
-        atomic_set_unordered(&thread->btrace, true);
-
-        thread_wakeup(thread);
+        irq_spinlock_lock(&thread->lock, true);
+
+        bool sleeping = false;
+        istate_t *istate = thread->udebug.uspace_state;
+        if (istate != NULL) {
+                printf("Scheduling thread stack trace.\n");
+                thread->btrace = true;
+                if (thread->state == Sleeping)
+                        sleeping = true;
+        } else
+                printf("Thread interrupt state not available.\n");
+
+        irq_spinlock_unlock(&thread->lock, true);
+
+        if (sleeping)
+                thread_wakeup(thread);
+
         thread_put(thread);
 }
@@ -1011 +1115 @@
                 thread_attach(thread, TASK);
 #endif
-                thread_start(thread);
-                thread_put(thread);
+                thread_ready(thread);
 
                 return 0;