Changeset f2cb80a in mainline for kernel

Timestamp:

2024-02-23T17:57:23Z (2 years ago)

Author:

GitHub <noreply@…>

Children:

192019f

Parents:

86f862c (diff), 90ba06c (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

git-author:

boba-buba <120932204+boba-buba@…> (2024-02-23 17:57:23)

git-committer:

GitHub <noreply@…> (2024-02-23 17:57:23)

Message:

Merge branch 'HelenOS:master' into topic/packet-capture

Location:

kernel

Files:

• arch/amd64/_link.ld.in (modified) (1 diff)
• arch/amd64/include/arch/mm/page.h (modified) (4 diffs)
• arch/amd64/include/arch/mm/pat.h (moved) (moved from uspace/lib/c/arch/amd64/include/libarch/stackarg.h ) (3 diffs)
• arch/amd64/src/amd64.c (modified) (2 diffs)
• arch/arm32/_link.ld.in (modified) (3 diffs)
• arch/arm64/_link.ld.in (modified) (2 diffs)
• arch/arm64/src/interrupt.c (modified) (1 diff)
• arch/ia32/_link.ld.in (modified) (1 diff)
• arch/ia32/include/arch/mm/page.h (modified) (2 diffs)
• arch/ia32/include/arch/mm/pat.h (added)
• arch/ia32/src/ia32.c (modified) (2 diffs)
• arch/ia64/_link.ld.in (modified) (2 diffs)
• arch/ia64/src/drivers/it.c (modified) (1 diff)
• arch/ia64/src/drivers/ski.c (modified) (1 diff)
• arch/mips32/_link.ld.in (modified) (2 diffs)
• arch/mips32/src/interrupt.c (modified) (1 diff)
• arch/ppc32/_link.ld.in (modified) (1 diff)
• arch/riscv64/_link.ld.in (modified) (1 diff)
• arch/sparc64/_link.ld.in (modified) (1 diff)
• arch/sparc64/src/drivers/niagara.c (modified) (1 diff)
• arch/sparc64/src/drivers/tick.c (modified) (1 diff)
• arch/sparc64/src/proc/sun4u/scheduler.c (modified) (1 diff)
• arch/sparc64/src/proc/sun4v/scheduler.c (modified) (1 diff)
• genarch/src/fb/fb.c (modified) (1 diff)
• genarch/src/kbrd/kbrd.c (modified) (1 diff)
• genarch/src/kbrd/kbrd_at.c (modified) (1 diff)
• genarch/src/srln/srln.c (modified) (1 diff)
• generic/include/atomic.h (modified) (1 diff)
• generic/include/context.h (modified) (3 diffs)
• generic/include/cpu.h (modified) (3 diffs)
• generic/include/lib/refcount.h (modified) (1 diff)
• generic/include/mm/mm.h (modified) (2 diffs)
• generic/include/proc/scheduler.h (modified) (2 diffs)
• generic/include/proc/task.h (modified) (2 diffs)
• generic/include/proc/thread.h (modified) (7 diffs)
• generic/meson.build (modified) (1 diff)
• generic/src/console/cmd.c (modified) (4 diffs)
• generic/src/cpu/cpu.c (modified) (2 diffs)
• generic/src/ddi/ddi.c (modified) (2 diffs)
• generic/src/interrupt/interrupt.c (modified) (2 diffs)
• generic/src/ipc/ipc.c (modified) (1 diff)
• generic/src/ipc/kbox.c (modified) (6 diffs)
• generic/src/main/kinit.c (modified) (5 diffs)
• generic/src/main/main.c (modified) (6 diffs)
• generic/src/preempt/preemption.c (deleted)
• generic/src/proc/program.c (modified) (4 diffs)
• generic/src/proc/scheduler.c (modified) (13 diffs)
• generic/src/proc/task.c (modified) (14 diffs)
• generic/src/proc/thread.c (modified) (28 diffs)
• generic/src/security/perm.c (modified) (3 diffs)
• generic/src/syscall/syscall.c (modified) (2 diffs)
• generic/src/sysinfo/stats.c (modified) (7 diffs)
• generic/src/time/clock.c (modified) (3 diffs)
• generic/src/time/timeout.c (modified) (1 diff)
• generic/src/udebug/udebug_ops.c (modified) (3 diffs)
• test/mm/falloc2.c (modified) (4 diffs)
• test/mm/slab1.c (modified) (2 diffs)
• test/mm/slab2.c (modified) (4 diffs)
• test/synch/semaphore1.c (modified) (1 diff)
• test/synch/semaphore2.c (modified) (1 diff)
• test/thread/thread1.c (modified) (3 diffs)

kernel/arch/amd64/_link.ld.in

@@ -23 +23 @@
         }
 
-        .mapped (PA2KA(BOOT_OFFSET) + SIZEOF_HEADERS + SIZEOF(.unmapped)) : AT (BOOT_OFFSET + SIZEOF_HEADERS + SIZEOF(.unmapped)) {
+        .text (PA2KA(BOOT_OFFSET) + SIZEOF_HEADERS + SIZEOF(.unmapped)) : AT (BOOT_OFFSET + SIZEOF_HEADERS + SIZEOF(.unmapped)) {
                 ktext_start = .;
                 *(.text .text.*);
                 ktext_end = .;
+        }
 
+        /* stack unwinding data */
+        .eh_frame_hdr : {
+                eh_frame_hdr_start = .;
+                *(.eh_frame_hdr) *(.eh_frame_entry .eh_frame_entry.*);
+                eh_frame_hdr_end = .;
+        }
+        .eh_frame : {
+                eh_frame_start = .;
+                KEEP(*(.eh_frame .eh_frame.*));
+                eh_frame_end = .;
+        }
+
+        .data : {
                 kdata_start = .;
-                *(.data .data.*);              /* initialized data */
-                *(.rodata .rodata.*);
-                *(.eh_frame .eh_frame.*);      /* stack unwinding data */
-                *(.eh_frame_hdr .eh_frame_hdr.*);
-                *(COMMON);      /* global variables */
+                *(.rodata .rodata.*);  /* read-only global variables */
+                *(.data .data.*);      /* non-zero initialized global variables */
 
-                /* bss can't be omitted from the ELF image. */
-                *(.bss .bss.*);        /* uninitialized static variables */
+                /*
+                 * When .bss is not physically present in the ELF file (MemSz > FileSz)
+                 * the kernel crashes during early boot. Not sure which part of the
+                 * boot process is to blame, for now just keep .bss packaged with .data
+                 * so that FileSz == MemSz.
+                 */
+
+                 *(.bss .bss.*);        /* uninitialized global variables */
+                *(COMMON);             /* non-`static` global variables without an extern declaration */
                 kdata_end = .;
         }

kernel/arch/amd64/include/arch/mm/page.h

@@ -192 +192 @@
         unsigned int accessed : 1;
         unsigned int dirty : 1;
-        unsigned int unused : 1;
+        unsigned int pat : 1;
         unsigned int global : 1;
         unsigned int soft_valid : 1;  /**< Valid content even if present bit is cleared. */
@@ -211 +211 @@
             p->writeable << PAGE_WRITE_SHIFT |
             (!p->no_execute) << PAGE_EXEC_SHIFT |
-            p->global << PAGE_GLOBAL_SHIFT);
+            p->global << PAGE_GLOBAL_SHIFT |
+            p->page_write_through << PAGE_WRITE_COMBINE_SHIFT);
 }
 
@@ -225 +226 @@
         pte_t *p = &pt[i];
 
-        p->page_cache_disable = !(flags & PAGE_CACHEABLE);
         p->present = !(flags & PAGE_NOT_PRESENT);
         p->uaccessible = (flags & PAGE_USER) != 0;
@@ -232 +232 @@
         p->global = (flags & PAGE_GLOBAL) != 0;
 
+        if (flags & PAGE_WRITE_COMBINE) {
+                /* We have mapped PCD+PWT bits to write-combine mode via PAT MSR. */
+                /* (If PAT is unsupported, it will default to uncached.) */
+                p->page_cache_disable = 1;
+                p->page_write_through = 1;
+        } else {
+                p->page_cache_disable = !(flags & PAGE_CACHEABLE);
+                p->page_write_through = 0;
+        }
+
         /*
          * Ensure that there is at least one bit set even if the present bit is cleared.

kernel/arch/amd64/include/arch/mm/pat.h

@@ -1 +1 @@
 /*
- * Copyright (c) 2006 Josef Cejka
+ * Copyright (c) 2024 Jiří Zárevúcky
  * All rights reserved.
  *
@@ -27 +27 @@
  */
 
-/** @addtogroup libcamd64
+/** @addtogroup kernel_amd64_mm
  * @{
  */
@@ -33 +33 @@
  */
 
-#ifndef _LIBC_STACKARG_H_
-#define _LIBC_STACKARG_H_
+#ifndef KERN_amd64_MM_PAT_H_
+#define KERN_amd64_MM_PAT_H_
+
+#include <arch/asm.h>
+#include <arch/cpuid.h>
+
+#define MSR_IA32_PAT  0x00000277
+
+typedef enum {
+        PAT_TYPE_UNCACHEABLE = 0,
+        PAT_TYPE_WRITE_COMBINING = 1,
+        PAT_TYPE_WRITE_THROUGH = 4,
+        PAT_TYPE_WRITE_PROTECTED = 5,
+        PAT_TYPE_WRITE_BACK = 6,
+        PAT_TYPE_UNCACHED  = 7,
+} pat_type_t;
+
+/**
+ * Assign caching type for a particular combination of PAT,
+ * PCD and PWT bits in PTE.
+ */
+static inline void pat_set_mapping(bool pat, bool pcd, bool pwt,
+    pat_type_t type)
+{
+        int index = pat << 2 | pcd << 1 | pwt;
+        int shift = index * 8;
+
+        uint64_t r = read_msr(MSR_IA32_PAT);
+        r &= ~(0xffull << shift);
+        r |= ((uint64_t) type) << shift;
+        write_msr(MSR_IA32_PAT, r);
+}
+
+static inline bool pat_supported(void)
+{
+        if (!has_cpuid())
+                return false;
+
+        cpu_info_t info;
+        cpuid(INTEL_CPUID_STANDARD, &info);
+
+        return (info.cpuid_edx & (1 << 16)) != 0;
+}
 
 #endif

kernel/arch/amd64/src/amd64.c

@@ -60 +60 @@
 #include <arch/vreg.h>
 #include <arch/kseg.h>
+#include <arch/mm/pat.h>
 #include <genarch/pic/pic_ops.h>
 
@@ -115 +116 @@
         /* Disable alignment check */
         write_cr0(read_cr0() & ~CR0_AM);
+
+        /* Use PCD+PWT bit combination in PTE to mean write-combining mode. */
+        if (pat_supported())
+                pat_set_mapping(false, true, true, PAT_TYPE_WRITE_COMBINING);
 
         if (config.cpu_active == 1) {

kernel/arch/arm32/_link.ld.in

@@ -26 +26 @@
         kernel_load_address = .;
         . = . + SIZEOF_HEADERS;
+
         .text : {
                 ktext_start = .;
@@ -31 +32 @@
                 ktext_end = .;
         }
+
+        /* stack unwinding data */
+        .eh_frame_hdr : {
+                eh_frame_hdr_start = .;
+                *(.eh_frame_hdr) *(.eh_frame_entry .eh_frame_entry.*);
+                eh_frame_hdr_end = .;
+        }
+
+        .eh_frame : {
+                eh_frame_start = .;
+                KEEP(*(.eh_frame .eh_frame.*));
+                eh_frame_end = .;
+        }
+
         .data : {
                 kdata_start = .;
@@ -38 +53 @@
 
                 *(.rodata .rodata.*);
-                *(.eh_frame .eh_frame.*);       /* stack unwinding data */
-                *(.eh_frame_hdr .eh_frame_hdr.*);
                 *(.sdata .sdata.*);
-        }
-        .sbss : {
                 *(.sbss .sbss.*);
                 *(.scommon .scommon.*);
+                kdata_end = .;
         }
-
-        kdata_end = .;
 
         .comment 0 : { *(.comment); }

kernel/arch/arm64/_link.ld.in

@@ -19 +19 @@
         kernel_load_address = LOAD_ADDRESS_V;
 
-        .image (LOAD_ADDRESS_V + SIZEOF_HEADERS) : AT (LOAD_ADDRESS_P + SIZEOF_HEADERS) {
+        .text (LOAD_ADDRESS_V + SIZEOF_HEADERS) : AT (LOAD_ADDRESS_P + SIZEOF_HEADERS) {
                 . = ALIGN(16);
                 ktext_start = .;
@@ -25 +25 @@
                 *(.text .text.*);
                 ktext_end = .;
+        }
 
+        /* stack unwinding data */
+        .eh_frame_hdr : {
+                eh_frame_hdr_start = .;
+                *(.eh_frame_hdr) *(.eh_frame_entry .eh_frame_entry.*);
+                eh_frame_hdr_end = .;
+        }
+
+        .eh_frame : {
+                eh_frame_start = .;
+                KEEP(*(.eh_frame .eh_frame.*));
+                eh_frame_end = .;
+        }
+
+        .data : {
                 kdata_start = .;
                 KEEP(*(K_DATA_START))
+                *(.rodata .rodata.*);
                 *(.data .data.*);                       /* initialized data */
                 *(.bss .bss.*);                        /* uninitialized static variables */
                 *(COMMON);                      /* global variables */
-
-                *(.rodata .rodata.*);
-                *(.eh_frame .eh_frame.*);       /* stack unwinding data */
-                *(.eh_frame_hdr .eh_frame_hdr.*);
-
                 kdata_end = .;
         }

kernel/arch/arm64/src/interrupt.c

@@ -137 +137 @@
         while (drift > timer_increment) {
                 drift -= timer_increment;
-                CPU->missed_clock_ticks++;
+                CPU_LOCAL->missed_clock_ticks++;
         }
         CNTV_CVAL_EL0_write(cntvct + timer_increment - drift);

kernel/arch/ia32/_link.ld.in

@@ -22 +22 @@
         }
 
-        .mapped (PA2KA(BOOT_OFFSET) + SIZEOF_HEADERS + SIZEOF(.unmapped)): AT (BOOT_OFFSET + SIZEOF_HEADERS + SIZEOF(.unmapped)) {
+        .text (PA2KA(BOOT_OFFSET) + SIZEOF_HEADERS + SIZEOF(.unmapped)): AT (BOOT_OFFSET + SIZEOF_HEADERS + SIZEOF(.unmapped)) {
                 ktext_start = .;
                 *(.text .text.*);
                 ktext_end = .;
+        }
 
+        /* stack unwinding data */
+        .eh_frame_hdr : {
+                eh_frame_hdr_start = .;
+                *(.eh_frame_hdr) *(.eh_frame_entry .eh_frame_entry.*);
+                eh_frame_hdr_end = .;
+        }
+        .eh_frame : {
+                eh_frame_start = .;
+                KEEP(*(.eh_frame .eh_frame.*));
+                eh_frame_end = .;
+        }
+
+        .data : {
                 kdata_start = .;
-                *(.data .data.*);               /* initialized data */
-                *(.rodata .rodata.*);
-                *(.eh_frame .eh_frame.*);       /* stack unwinding data */
-                *(.eh_frame_hdr .eh_frame_hdr.*);
-                *(COMMON);              /* global variables */
+                *(.rodata .rodata.*);  /* read-only global variables */
+                *(.data .data.*);      /* non-zero initialized global variables */
 
-                /* bss can't be omitted from the ELF image. */
-                *(.bss .bss.*);                /* uninitialized static variables */
+                /*
+                 * When .bss is not physically present in the ELF file (MemSz > FileSz)
+                 * the kernel crashes during early boot. Not sure which part of the
+                 * boot process is to blame, for now just keep .bss packaged with .data
+                 * so that FileSz == MemSz.
+                 */
+
+                 *(.bss .bss.*);        /* uninitialized global variables */
+                *(COMMON);             /* non-`static` global variables without an extern declaration */
                 kdata_end = .;
         }

kernel/arch/ia32/include/arch/mm/page.h

@@ -190 +190 @@
             p->writeable << PAGE_WRITE_SHIFT |
             1 << PAGE_EXEC_SHIFT |
-            p->global << PAGE_GLOBAL_SHIFT);
+            p->global << PAGE_GLOBAL_SHIFT |
+            p->page_write_through << PAGE_WRITE_COMBINE_SHIFT);
 }
 
@@ -197 +198 @@
         pte_t *p = &pt[i];
 
-        p->page_cache_disable = !(flags & PAGE_CACHEABLE);
         p->present = !(flags & PAGE_NOT_PRESENT);
         p->uaccessible = (flags & PAGE_USER) != 0;
         p->writeable = (flags & PAGE_WRITE) != 0;
         p->global = (flags & PAGE_GLOBAL) != 0;
+
+        if (flags & PAGE_WRITE_COMBINE) {
+                /* We have mapped PCD+PWT bits to write-combine mode via PAT MSR. */
+                /* (If PAT is unsupported, it will default to uncached.) */
+                p->page_cache_disable = 1;
+                p->page_write_through = 1;
+        } else {
+                p->page_cache_disable = !(flags & PAGE_CACHEABLE);
+                p->page_write_through = 0;
+        }
 
         /*

kernel/arch/ia32/src/ia32.c

@@ -61 +61 @@
 #include <arch/pm.h>
 #include <arch/vreg.h>
+#include <arch/mm/pat.h>
 
 #ifdef CONFIG_SMP
@@ -104 +105 @@
 {
         pm_init();
+
+        /* Use PCD+PWT bit combination in PTE to mean write-combining mode. */
+        if (pat_supported())
+                pat_set_mapping(false, true, true, PAT_TYPE_WRITE_COMBINING);
 
         if (config.cpu_active == 1) {

kernel/arch/ia64/_link.ld.in

@@ -15 +15 @@
         kernel_load_address = LOAD_ADDRESS_V;
 
-        .image (LOAD_ADDRESS_V + SIZEOF_HEADERS): AT (LOAD_ADDRESS_P + SIZEOF_HEADERS) {
+        .text (LOAD_ADDRESS_V + SIZEOF_HEADERS): AT (LOAD_ADDRESS_P + SIZEOF_HEADERS) {
                 . = ALIGN(16);
                 ktext_start = .;
@@ -21 +21 @@
                 *(.text .text.*)
                 ktext_end = .;
+        }
 
+        /* stack unwinding data */
+        .eh_frame_hdr : {
+                eh_frame_hdr_start = .;
+                *(.eh_frame_hdr) *(.eh_frame_entry .eh_frame_entry.*);
+                eh_frame_hdr_end = .;
+        }
+
+        .eh_frame : {
+                eh_frame_start = .;
+                KEEP(*(.eh_frame .eh_frame.*));
+                eh_frame_end = .;
+        }
+        
+        .data : {
                 kdata_start = .;
                 KEEP(*(K_DATA_START));
                 *(.rodata .rodata.*);
-                *(.eh_frame .eh_frame.*);       /* stack unwinding data */
-                *(.eh_frame_hdr .eh_frame_hdr.*);
                 *(.opd)
                 *(.data .data.*)

kernel/arch/ia64/src/drivers/it.c

@@ -122 +122 @@
                 itm += IT_DELTA;
                 if (itm - itc < 0)
-                        CPU->missed_clock_ticks++;
+                        CPU_LOCAL->missed_clock_ticks++;
                 else
                         break;

kernel/arch/ia64/src/drivers/ski.c

@@ -258 +258 @@
 
         instance->srlnin = srlnin;
-        thread_ready(instance->thread);
+        thread_start(instance->thread);
 
         sysinfo_set_item_val("kbd", NULL, true);

kernel/arch/mips32/_link.ld.in

@@ -33 +33 @@
                 ktext_end = .;
         }
+
+        /* stack unwinding data */
+        .eh_frame_hdr : {
+                eh_frame_hdr_start = .;
+                *(.eh_frame_hdr) *(.eh_frame_entry .eh_frame_entry.*);
+                eh_frame_hdr_end = .;
+        }
+
+        .eh_frame : {
+                eh_frame_start = .;
+                KEEP(*(.eh_frame .eh_frame.*));
+                eh_frame_end = .;
+        }
+
         .data : {
                 kdata_start = .;
                 *(.data .data.*);                       /* initialized data */
                 *(.rodata .rodata.*);
-                *(.eh_frame .eh_frame.*);       /* stack unwinding data */
-                *(.eh_frame_hdr .eh_frame_hdr.*);
                 *(.sdata .sdata.*);
                 *(.reginfo);
@@ -46 +58 @@
                 *(COMMON);                      /* global variables */
         }
+
         _gp = . + 0x8000;
         .lit8 : { *(.lit8) }

kernel/arch/mips32/src/interrupt.c

@@ -121 +121 @@
         while (drift > cp0_compare_value) {
                 drift -= cp0_compare_value;
-                CPU->missed_clock_ticks++;
+                CPU_LOCAL->missed_clock_ticks++;
         }
 

kernel/arch/ppc32/_link.ld.in

@@ -25 +25 @@
         }
 
-        .mapped PA2KA(BOOT_OFFSET): AT (BOOT_OFFSET) {
+        .text PA2KA(BOOT_OFFSET): AT (BOOT_OFFSET) {
                 ktext_start = .;
                 KEEP(*(K_TEXT_START));
                 *(.text .text.*);
                 ktext_end = .;
+        }
 
+        /* stack unwinding data */
+        .eh_frame_hdr : {
+                eh_frame_hdr_start = .;
+                *(.eh_frame_hdr) *(.eh_frame_entry .eh_frame_entry.*);
+                eh_frame_hdr_end = .;
+        }
+
+        .eh_frame : {
+                eh_frame_start = .;
+                KEEP(*(.eh_frame .eh_frame.*));
+                eh_frame_end = .;
+        }
+
+        .data : {
                 kdata_start = .;
                 KEEP(*(K_DATA_START));
                 *(.rodata .rodata.*);
-                *(.eh_frame .eh_frame.*);       /* stack unwinding data */
-                *(.eh_frame_hdr .eh_frame_hdr.*);
                 *(.data .data.*);       /* initialized data */
                 *(.sdata .sdata.*);

kernel/arch/riscv64/_link.ld.in

@@ -16 +16 @@
         kernel_load_address = PA2KA(BOOT_OFFSET);
 
-        .image (PA2KA(BOOT_OFFSET) + SIZEOF_HEADERS) : AT (BOOT_OFFSET + SIZEOF_HEADERS) {
+        .text (PA2KA(BOOT_OFFSET) + SIZEOF_HEADERS) : AT (BOOT_OFFSET + SIZEOF_HEADERS) {
                 ktext_start = .;
                 KEEP(*(K_TEXT_START));
                 *(.text .text.*);
                 ktext_end = .;
+        }
 
+        /* stack unwinding data */
+        .eh_frame_hdr : {
+                eh_frame_hdr_start = .;
+                *(.eh_frame_hdr) *(.eh_frame_entry .eh_frame_entry.*);
+                eh_frame_hdr_end = .;
+        }
+
+        .eh_frame : {
+                eh_frame_start = .;
+                KEEP(*(.eh_frame .eh_frame.*));
+                eh_frame_end = .;
+        }
+
+        .data : {
                 kdata_start = .;
                 *(.data .data.*);                       /* initialized data */
                 *(.rodata .rodata.*);
-                *(.eh_frame .eh_frame.*);       /* stack unwinding data */
-                *(.eh_frame_hdr .eh_frame_hdr.*);
                 *(.sdata .sdata.*);
                 *(.sbss .sbss.*);

kernel/arch/sparc64/_link.ld.in

@@ -14 +14 @@
         kernel_load_address = VMA;
 
-        .image (VMA + SIZEOF_HEADERS): AT (LMA + SIZEOF_HEADERS) {
+        .text (VMA + SIZEOF_HEADERS): AT (LMA + SIZEOF_HEADERS) {
                 ktext_start = .;
                 KEEP(*(K_TEXT_START));
                 *(.text .text.*);
                 ktext_end = .;
+        }
 
+        /* stack unwinding data */
+        .eh_frame_hdr : {
+                eh_frame_hdr_start = .;
+                *(.eh_frame_hdr) *(.eh_frame_entry .eh_frame_entry.*);
+                eh_frame_hdr_end = .;
+        }
+
+        .eh_frame : {
+                eh_frame_start = .;
+                KEEP(*(.eh_frame .eh_frame.*));
+                eh_frame_end = .;
+        }
+
+        .data : {
                 kdata_start = .;
                 KEEP(*(K_DATA_START));
                 *(.rodata .rodata.*);
-                *(.eh_frame .eh_frame.*);       /* stack unwinding data */
-                *(.eh_frame_hdr .eh_frame_hdr.*);
                 *(.data .data.*);                   /* initialized data */
                 *(.sdata .sdata.*);

kernel/arch/sparc64/src/drivers/niagara.c

@@ -253 +253 @@
 
                         instance->srlnin = srln;
-                        thread_ready(instance->thread);
+                        thread_start(instance->thread);
                 }
         }

kernel/arch/sparc64/src/drivers/tick.c

@@ -117 +117 @@
         while (drift > CPU->arch.clock_frequency / HZ) {
                 drift -= CPU->arch.clock_frequency / HZ;
-                CPU->missed_clock_ticks++;
+                CPU_LOCAL->missed_clock_ticks++;
         }
         CPU->arch.next_tick_cmpr = tick_counter_read() +

kernel/arch/sparc64/src/proc/sun4u/scheduler.c

@@ -76 +76 @@
 {
         if (THREAD->uspace) {
+                asm volatile ("flushw");
+
                 /* sample the state of the userspace window buffer */
                 THREAD->arch.uspace_window_buffer =

kernel/arch/sparc64/src/proc/sun4v/scheduler.c

@@ -68 +68 @@
 {
         if (THREAD->uspace) {
+                asm volatile ("flushw");
+
                 /* sample the state of the userspace window buffer */
                 THREAD->arch.uspace_window_buffer =

kernel/genarch/src/fb/fb.c

@@ -633 +633 @@
 
         instance->addr = (uint8_t *) km_map((uintptr_t) props->addr, fbsize,
-            KM_NATURAL_ALIGNMENT, PAGE_WRITE | PAGE_NOT_CACHEABLE);
+            KM_NATURAL_ALIGNMENT, PAGE_WRITE | PAGE_WRITE_COMBINE);
         if (!instance->addr) {
                 LOG("Unable to map framebuffer.");

kernel/genarch/src/kbrd/kbrd.c

@@ -200 +200 @@
 
         instance->sink = sink;
-        thread_ready(instance->thread);
+        thread_start(instance->thread);
 
         return &instance->raw;

kernel/genarch/src/kbrd/kbrd_at.c

@@ -198 +198 @@
 
         instance->sink = sink;
-        thread_ready(instance->thread);
+        thread_start(instance->thread);
 
         return &instance->raw;

kernel/genarch/src/srln/srln.c

@@ -156 +156 @@
 
         instance->sink = sink;
-        thread_ready(instance->thread);
+        thread_start(instance->thread);
 
         return &instance->raw;

kernel/generic/include/atomic.h

@@ -39 +39 @@
 #include <typedefs.h>
 #include <stdatomic.h>
+
+/*
+ * Shorthand for relaxed atomic read/write, something that's needed to formally
+ * avoid undefined behavior in cases where we need to read a variable in
+ * different threads and we don't particularly care about ordering
+ * (e.g. statistic printouts). This is most likely translated into the same
+ * assembly instructions as regular read/writes.
+ */
+#define atomic_set_unordered(var, val) atomic_store_explicit((var), (val), memory_order_relaxed)
+#define atomic_get_unordered(var) atomic_load_explicit((var), memory_order_relaxed)
 
 #define atomic_predec(val) \

kernel/generic/include/context.h

@@ -36 +36 @@
 #define KERN_CONTEXT_H_
 
+#include <panic.h>
 #include <trace.h>
 #include <arch/context.h>
+#include <arch/faddr.h>
 
 #define context_set_generic(ctx, _pc, stack, size) \
@@ -47 +49 @@
 extern int context_save_arch(context_t *ctx) __attribute__((returns_twice));
 extern void context_restore_arch(context_t *ctx) __attribute__((noreturn));
-
-/** Save register context.
- *
- * Save the current register context (including stack pointer) to a context
- * structure. A subsequent call to context_restore() will return to the same
- * address as the corresponding call to context_save().
- *
- * Note that context_save_arch() must reuse the stack frame of the function
- * which called context_save(). We guarantee this by:
- *
- *   a) implementing context_save_arch() in assembly so that it does not create
- *      its own stack frame, and by
- *   b) defining context_save() as a macro because the inline keyword is just a
- *      hint for the compiler, not a real constraint; the application of a macro
- *      will definitely not create a stack frame either.
- *
- * To imagine what could happen if there were some extra stack frames created
- * either by context_save() or context_save_arch(), we need to realize that the
- * sp saved in the contex_t structure points to the current stack frame as it
- * existed when context_save_arch() was executing. After the return from
- * context_save_arch() and context_save(), any extra stack frames created by
- * these functions will be destroyed and their contents sooner or later
- * overwritten by functions called next. Any attempt to restore to a context
- * saved like that would therefore lead to a disaster.
- *
- * @param ctx Context structure.
- *
- * @return context_save() returns 1, context_restore() returns 0.
- *
- */
-#define context_save(ctx)  context_save_arch(ctx)
 
 /** Restore register context.
@@ -91 +62 @@
  *
  */
-_NO_TRACE static inline void context_restore(context_t *ctx)
+_NO_TRACE __attribute__((noreturn))
+    static inline void context_restore(context_t *ctx)
 {
         context_restore_arch(ctx);
+}
+
+/**
+ * Saves current context to the variable pointed to by `self`,
+ * and restores the context denoted by `other`.
+ *
+ * When the `self` context is later restored by another call to
+ * `context_swap()`, the control flow behaves as if the earlier call to
+ * `context_swap()` just returned.
+ */
+_NO_TRACE static inline void context_swap(context_t *self, context_t *other)
+{
+        if (context_save_arch(self))
+                context_restore_arch(other);
+}
+
+_NO_TRACE static inline void context_create(context_t *context,
+    void (*fn)(void), void *stack_base, size_t stack_size)
+{
+        *context = (context_t) { 0 };
+        context_set(context, FADDR(fn), stack_base, stack_size);
+}
+
+__attribute__((noreturn)) static inline void context_replace(void (*fn)(void),
+    void *stack_base, size_t stack_size)
+{
+        context_t ctx;
+        context_create(&ctx, fn, stack_base, stack_size);
+        context_restore(&ctx);
 }
 

kernel/generic/include/cpu.h

@@ -44 +44 @@
 #include <arch.h>
 
-#define CPU                  CURRENT->cpu
+#define CPU                  (CURRENT->cpu)
+#define CPU_LOCAL            (&CPU->local)
+
+/**
+ * Contents of CPU_LOCAL. These are variables that are only ever accessed by
+ * the CPU they belong to, so they don't need any synchronization,
+ * just locally disabled interrupts.
+ */
+typedef struct cpu_local {
+        /**
+         * When system clock loses a tick, it is
+         * recorded here so that clock() can react.
+         */
+        size_t missed_clock_ticks;
+
+        uint64_t current_clock_tick;
+        uint64_t preempt_deadline;  /* < when should the currently running thread be preempted */
+        uint64_t relink_deadline;
+
+        /**
+         * Stack used by scheduler when there is no running thread.
+         * This field is unchanged after initialization.
+         */
+        uint8_t *stack;
+
+        /**
+         * Processor cycle accounting.
+         */
+        bool idle;
+        uint64_t last_cycle;
+
+        context_t scheduler_context;
+
+        struct thread *prev_thread;
+} cpu_local_t;
 
 /** CPU structure.
@@ -63 +97 @@
 
         /**
-         * When system clock loses a tick, it is
-         * recorded here so that clock() can react.
-         * This variable is CPU-local and can be
-         * only accessed when interrupts are
-         * disabled.
-         */
-        size_t missed_clock_ticks;
-
-        /** Can only be accessed by the CPU represented by this structure when interrupts are disabled. */
-        uint64_t current_clock_tick;
-        uint64_t preempt_deadline;  /* < when should the currently running thread be preempted */
-        uint64_t relink_deadline;
-
-        /**
          * Processor cycle accounting.
          */
-        bool idle;
-        uint64_t last_cycle;
         atomic_time_stat_t idle_cycles;
         atomic_time_stat_t busy_cycles;
@@ -103 +121 @@
         _Atomic(struct thread *) fpu_owner;
 
-        /**
-         * Stack used by scheduler when there is no running thread.
-         */
-        uint8_t *stack;
+        cpu_local_t local;
 } cpu_t;
 

kernel/generic/include/lib/refcount.h

@@ -49 +49 @@
 } atomic_refcount_t;
 
+#define REFCOUNT_INITIALIZER() { \
+        .__cnt = ATOMIC_VAR_INIT(0), \
+}
+
 static inline void refcount_init(atomic_refcount_t *rc)
 {
-        atomic_store_explicit(&rc->__cnt, 0, memory_order_relaxed);
+        atomic_init(&rc->__cnt, 0);
 }
 

kernel/generic/include/mm/mm.h

@@ -46 +46 @@
 #define PAGE_EXEC_SHIFT                 5
 #define PAGE_GLOBAL_SHIFT               6
+#define PAGE_WRITE_COMBINE_SHIFT  7
 
 #define PAGE_NOT_CACHEABLE              (0 << PAGE_CACHEABLE_SHIFT)
@@ -62 +63 @@
 #define PAGE_GLOBAL                     (1 << PAGE_GLOBAL_SHIFT)
 
+#define PAGE_WRITE_COMBINE  (1 << PAGE_WRITE_COMBINE_SHIFT)
+
 #endif
 

kernel/generic/include/proc/scheduler.h

@@ -41 +41 @@
 #include <atomic.h>
 #include <adt/list.h>
+#include <abi/proc/thread.h>
 
 #define RQ_COUNT          16
@@ -56 +57 @@
 
 extern void scheduler_fpu_lazy_request(void);
-extern void scheduler(void);
-extern void scheduler_locked(ipl_t);
 extern void kcpulb(void *arg);
 
 extern void sched_print_list(void);
+
+extern void scheduler_run(void) __attribute__((noreturn));
+extern void scheduler_enter(state_t);
+
+extern void thread_main_func(void);
 
 /*

kernel/generic/include/proc/task.h

@@ -88 +88 @@
 
         /** Number of references (i.e. threads). */
-        atomic_size_t refcount;
+        atomic_refcount_t refcount;
         /** Number of threads that haven't exited yet. */
         // TODO: remove
@@ -144 +144 @@
 extern void task_done(void);
 extern task_t *task_create(as_t *, const char *);
-extern void task_destroy(task_t *);
 extern void task_hold(task_t *);
 extern void task_release(task_t *);

kernel/generic/include/proc/thread.h

@@ -95 +95 @@
         waitq_t join_wq;
 
-        /** Lock protecting thread structure.
+        /** Thread accounting. */
+        atomic_time_stat_t ucycles;
+        atomic_time_stat_t kcycles;
+
+        /** Architecture-specific data. */
+        thread_arch_t arch;
+
+#ifdef CONFIG_UDEBUG
+        /**
+         * If true, the scheduler will print a stack trace
+         * to the kernel console upon scheduling this thread.
+         */
+        atomic_int_fast8_t btrace;
+
+        /** Debugging stuff */
+        udebug_thread_t udebug;
+#endif /* CONFIG_UDEBUG */
+
+        /*
+         * Immutable fields.
          *
-         * Protects the whole thread structure except fields listed above.
-         */
-        IRQ_SPINLOCK_DECLARE(lock);
-
-        char name[THREAD_NAME_BUFLEN];
+         * These fields are only modified during initialization, and are not
+         * changed at any time between initialization and destruction.
+         * Can be accessed without synchronization in most places.
+         */
+
+        /** Thread ID. */
+        thread_id_t tid;
 
         /** Function implementing the thread. */
@@ -108 +129 @@
         void *thread_arg;
 
+        char name[THREAD_NAME_BUFLEN];
+
+        /** Thread is executed in user space. */
+        bool uspace;
+
+        /** Thread doesn't affect accumulated accounting. */
+        bool uncounted;
+
+        /** Containing task. */
+        task_t *task;
+
+        /** Thread's kernel stack. */
+        uint8_t *kstack;
+
+        /*
+         * Local fields.
+         *
+         * These fields can be safely accessed from code that _controls execution_
+         * of this thread. Code controls execution of a thread if either:
+         *  - it runs in the context of said thread AND interrupts are disabled
+         *    (interrupts can and will access these fields)
+         *  - the thread is not running, and the code accessing it can legally
+         *    add/remove the thread to/from a runqueue, i.e., either:
+         *    - it is allowed to enqueue thread in a new runqueue
+         *    - it holds the lock to the runqueue containing the thread
+         *
+         */
+
         /**
          * From here, the stored context is restored
@@ -113 +162 @@
          */
         context_t saved_context;
-        ipl_t saved_ipl;
+
+        // TODO: we only need one of the two bools below
 
         /**
@@ -127 +177 @@
         bool in_copy_to_uspace;
 
+        /*
+         * FPU context is a special case. If lazy FPU switching is disabled,
+         * it acts as a regular local field. However, if lazy switching is enabled,
+         * the context is synchronized via CPU->fpu_lock
+         */
 #ifdef CONFIG_FPU
         fpu_context_t fpu_context;
@@ -135 +190 @@
         unsigned int nomigrate;
 
-        /** Thread state. */
-        state_t state;
-
-        /** Thread CPU. */
-        cpu_t *cpu;
-        /** Containing task. */
-        task_t *task;
         /** Thread was migrated to another CPU and has not run yet. */
         bool stolen;
-        /** Thread is executed in user space. */
-        bool uspace;
-
-        /** Thread accounting. */
-        uint64_t ucycles;
-        uint64_t kcycles;
+
+        /**
+         * Thread state (state_t).
+         * This is atomic because we read it via some commands for debug output,
+         * otherwise it could just be a regular local.
+         */
+        atomic_int_fast32_t state;
+
+        /** Thread CPU. */
+        _Atomic(cpu_t *) cpu;
+
+        /** Thread's priority. Implemented as index to CPU->rq */
+        atomic_int_fast32_t priority;
+
         /** Last sampled cycle. */
         uint64_t last_cycle;
-        /** Thread doesn't affect accumulated accounting. */
-        bool uncounted;
-
-        /** Thread's priority. Implemented as index to CPU->rq */
-        int priority;
-        /** Thread ID. */
-        thread_id_t tid;
-
-        /** Architecture-specific data. */
-        thread_arch_t arch;
-
-        /** Thread's kernel stack. */
-        uint8_t *kstack;
-
-#ifdef CONFIG_UDEBUG
-        /**
-         * If true, the scheduler will print a stack trace
-         * to the kernel console upon scheduling this thread.
-         */
-        bool btrace;
-
-        /** Debugging stuff */
-        udebug_thread_t udebug;
-#endif /* CONFIG_UDEBUG */
 } thread_t;
 
@@ -186 +218 @@
 extern void thread_wire(thread_t *, cpu_t *);
 extern void thread_attach(thread_t *, task_t *);
-extern void thread_ready(thread_t *);
+extern void thread_start(thread_t *);
+extern void thread_requeue_sleeping(thread_t *);
 extern void thread_exit(void) __attribute__((noreturn));
 extern void thread_interrupt(thread_t *);
+
+enum sleep_state {
+        SLEEP_INITIAL,
+        SLEEP_ASLEEP,
+        SLEEP_WOKE,
+};
 
 typedef enum {
@@ -237 +276 @@
 extern errno_t thread_join(thread_t *);
 extern errno_t thread_join_timeout(thread_t *, uint32_t, unsigned int);
+extern void thread_detach(thread_t *);
+
+extern void thread_yield(void);
 
 extern void thread_print_list(bool);

kernel/generic/meson.build

@@ -95 +95 @@
         'src/mm/malloc.c',
         'src/mm/reserve.c',
-        'src/preempt/preemption.c',
         'src/printf/printf.c',
         'src/printf/snprintf.c',

kernel/generic/src/console/cmd.c

@@ -229 +229 @@
 };
 
+/* Data and methods for 'printbench' command. */
+static int cmd_printbench(cmd_arg_t *argv);
+
+static cmd_info_t printbench_info = {
+        .name = "printbench",
+        .description = "Run a printing benchmark.",
+        .func = cmd_printbench,
+        .argc = 0,
+};
+
 #endif /* CONFIG_TEST */
 
@@ -613 +623 @@
         &test_info,
         &bench_info,
+        &printbench_info,
 #endif
 #ifdef CONFIG_UDEBUG
@@ -993 +1004 @@
                         printf("cpu%u: ", i);
                         thread_wire(thread, &cpus[i]);
-                        thread_ready(thread_ref(thread));
+                        thread_start(thread);
                         thread_join(thread);
-                        thread_put(thread);
                 } else
                         printf("Unable to create thread for cpu%u\n", i);
@@ -1582 +1592 @@
 }
 
+int cmd_printbench(cmd_arg_t *argv)
+{
+        int cnt = 20;
+
+        uint64_t *data = malloc(sizeof(uint64_t) * cnt);
+        if (data == NULL) {
+                printf("Error allocating memory for statistics\n");
+                return false;
+        }
+
+        for (int i = 0; i < cnt; i++) {
+                /*
+                 * Update and read thread accounting
+                 * for benchmarking
+                 */
+                irq_spinlock_lock(&TASK->lock, true);
+                uint64_t ucycles0, kcycles0;
+                task_get_accounting(TASK, &ucycles0, &kcycles0);
+                irq_spinlock_unlock(&TASK->lock, true);
+
+                /* Execute the test */
+                for (int j = 0; j < 20; j++) {
+                        printf("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ěščřžýáíéú!@#$%%^&*(){}+\n");
+                        printf("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789ěščřžýáíéú!@#$%%^&*(){}+abcdefghijklmnopqrstuvwxyz\n");
+                        printf("0123456789ěščřžýáíéú!@#$%%^&*(){}+abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\n");
+                }
+
+                /* Update and read thread accounting */
+                irq_spinlock_lock(&TASK->lock, true);
+                uint64_t ucycles1, kcycles1;
+                task_get_accounting(TASK, &ucycles1, &kcycles1);
+                irq_spinlock_unlock(&TASK->lock, true);
+
+                data[i] = ucycles1 - ucycles0 + kcycles1 - kcycles0;
+        }
+
+        printf("\n");
+
+        uint64_t cycles;
+        char suffix;
+        uint64_t sum = 0;
+
+        for (int i = 0; i < cnt; i++) {
+                sum += data[i];
+        }
+
+        order_suffix(sum / (uint64_t) cnt, &cycles, &suffix);
+        printf("Average\t\t%" PRIu64 "%c\n", cycles, suffix);
+
+        free(data);
+
+        return true;
+}
+
 #endif
 

kernel/generic/src/cpu/cpu.c

@@ -81 +81 @@
                                 panic("Cannot allocate CPU stack.");
 
-                        cpus[i].stack = (uint8_t *) PA2KA(stack_phys);
+                        cpus[i].local.stack = (uint8_t *) PA2KA(stack_phys);
                         cpus[i].id = i;
 
@@ -104 +104 @@
         CPU->tlb_active = true;
 
-        CPU->idle = false;
-        CPU->last_cycle = get_cycle();
+        CPU_LOCAL->idle = false;
+        CPU_LOCAL->last_cycle = get_cycle();
         CPU->idle_cycles = ATOMIC_TIME_INITIALIZER();
         CPU->busy_cycles = ATOMIC_TIME_INITIALIZER();

kernel/generic/src/ddi/ddi.c

@@ -336 +336 @@
                 return EPERM;
 
-        irq_spinlock_lock(&tasks_lock, true);
-
         task_t *task = task_find_by_id(id);
 
-        if ((!task) || (!container_check(CONTAINER, task->container))) {
-                /*
-                 * There is no task with the specified ID
-                 * or the task belongs to a different security
-                 * context.
-                 */
-                irq_spinlock_unlock(&tasks_lock, true);
+        if (!task)
                 return ENOENT;
-        }
-
-        /* Lock the task and release the lock protecting tasks dictionary. */
-        irq_spinlock_exchange(&tasks_lock, &task->lock);
-        errno_t rc = ddi_iospace_enable_arch(task, ioaddr, size);
+
+        errno_t rc = ENOENT;
+
+        irq_spinlock_lock(&task->lock, true);
+
+        /* Check that the task belongs to the correct security context. */
+        if (container_check(CONTAINER, task->container))
+                rc = ddi_iospace_enable_arch(task, ioaddr, size);
+
         irq_spinlock_unlock(&task->lock, true);
-
+        task_release(task);
         return rc;
 }
@@ -377 +373 @@
                 return EPERM;
 
-        irq_spinlock_lock(&tasks_lock, true);
-
         task_t *task = task_find_by_id(id);
 
-        if ((!task) || (!container_check(CONTAINER, task->container))) {
-                /*
-                 * There is no task with the specified ID
-                 * or the task belongs to a different security
-                 * context.
-                 */
-                irq_spinlock_unlock(&tasks_lock, true);
+        if (!task)
                 return ENOENT;
-        }
-
-        /* Lock the task and release the lock protecting tasks dictionary. */
-        irq_spinlock_exchange(&tasks_lock, &task->lock);
-        errno_t rc = ddi_iospace_disable_arch(task, ioaddr, size);
+
+        errno_t rc = ENOENT;
+
+        irq_spinlock_lock(&task->lock, true);
+
+        /* Check that the task belongs to the correct security context. */
+        if (container_check(CONTAINER, task->container))
+                rc = ddi_iospace_disable_arch(task, ioaddr, size);
+
         irq_spinlock_unlock(&task->lock, true);
-
+        task_release(task);
         return rc;
 }

kernel/generic/src/interrupt/interrupt.c

@@ -114 +114 @@
 
         /* Account user cycles */
-        if (THREAD) {
-                irq_spinlock_lock(&THREAD->lock, false);
+        if (THREAD)
                 thread_update_accounting(true);
-                irq_spinlock_unlock(&THREAD->lock, false);
-        }
 
         /* Account CPU usage if it woke up from sleep */
-        if (CPU && CPU->idle) {
+        if (CPU && CPU_LOCAL->idle) {
                 uint64_t now = get_cycle();
-                atomic_time_increment(&CPU->idle_cycles, now - CPU->last_cycle);
-                CPU->last_cycle = now;
-                CPU->idle = false;
+                atomic_time_increment(&CPU->idle_cycles, now - CPU_LOCAL->last_cycle);
+                CPU_LOCAL->last_cycle = now;
+                CPU_LOCAL->idle = false;
         }
 
@@ -155 +152 @@
 
         /* Do not charge THREAD for exception cycles */
-        if (THREAD) {
-                irq_spinlock_lock(&THREAD->lock, false);
+        if (THREAD)
                 THREAD->last_cycle = end_cycle;
-                irq_spinlock_unlock(&THREAD->lock, false);
-        }
 #else
         panic("No space for any exception handler, yet we want to handle some exception.");

kernel/generic/src/ipc/ipc.c

@@ -967 +967 @@
 void ipc_print_task(task_id_t taskid)
 {
-        irq_spinlock_lock(&tasks_lock, true);
         task_t *task = task_find_by_id(taskid);
-        if (!task) {
-                irq_spinlock_unlock(&tasks_lock, true);
+        if (!task)
                 return;
-        }
-        task_hold(task);
-        irq_spinlock_unlock(&tasks_lock, true);
 
         printf("[phone cap] [calls] [state\n");

kernel/generic/src/ipc/kbox.c

@@ -90 +90 @@
                 LOG("Join kb.thread.");
                 thread_join(TASK->kb.thread);
-                thread_put(TASK->kb.thread);
                 LOG("...join done.");
                 TASK->kb.thread = NULL;
@@ -201 +200 @@
 /** Connect phone to a task kernel-box specified by id.
  *
- * Note that this is not completely atomic. For optimisation reasons, the task
- * might start cleaning up kbox after the phone has been connected and before
- * a kbox thread has been created. This must be taken into account in the
- * cleanup code.
- *
  * @param[out] out_phone  Phone capability handle on success.
  * @return Error code.
@@ -212 +206 @@
 errno_t ipc_connect_kbox(task_id_t taskid, cap_phone_handle_t *out_phone)
 {
-        irq_spinlock_lock(&tasks_lock, true);
-
         task_t *task = task_find_by_id(taskid);
-        if (task == NULL) {
-                irq_spinlock_unlock(&tasks_lock, true);
+        if (!task)
                 return ENOENT;
-        }
-
-        atomic_inc(&task->refcount);
-
-        irq_spinlock_unlock(&tasks_lock, true);
 
         mutex_lock(&task->kb.cleanup_lock);
-
-        if (atomic_predec(&task->refcount) == 0) {
-                mutex_unlock(&task->kb.cleanup_lock);
-                task_destroy(task);
-                return ENOENT;
-        }
 
         if (task->kb.finished) {
                 mutex_unlock(&task->kb.cleanup_lock);
+                task_release(task);
                 return EINVAL;
         }
@@ -244 +225 @@
                 if (!kb_thread) {
                         mutex_unlock(&task->kb.cleanup_lock);
+                        task_release(task);
                         return ENOMEM;
                 }
 
-                task->kb.thread = thread_ref(kb_thread);
-                thread_ready(kb_thread);
+                task->kb.thread = kb_thread;
+                thread_start(kb_thread);
         }
 
@@ -256 +238 @@
         if (rc != EOK) {
                 mutex_unlock(&task->kb.cleanup_lock);
+                task_release(task);
                 return rc;
         }
@@ -266 +249 @@
 
         mutex_unlock(&task->kb.cleanup_lock);
+        task_release(task);
         *out_phone = phone_handle;
         return EOK;

kernel/generic/src/main/kinit.c

@@ -122 +122 @@
 
                 thread_wire(thread, &cpus[0]);
-                thread_ready(thread_ref(thread));
+                thread_start(thread);
                 thread_join(thread);
-                thread_put(thread);
 
                 /*
@@ -136 +135 @@
                         if (thread != NULL) {
                                 thread_wire(thread, &cpus[i]);
-                                thread_ready(thread);
+                                thread_start(thread);
+                                thread_detach(thread);
                         } else
                                 log(LF_OTHER, LVL_ERROR,
@@ -152 +152 @@
         thread = thread_create(kload, NULL, TASK, THREAD_FLAG_NONE,
             "kload");
-        if (thread != NULL)
-                thread_ready(thread);
-        else
+        if (thread != NULL) {
+                thread_start(thread);
+                thread_detach(thread);
+        } else {
                 log(LF_OTHER, LVL_ERROR, "Unable to create kload thread");
+        }
 
 #ifdef CONFIG_KCONSOLE
@@ -164 +166 @@
                 thread = thread_create(kconsole_thread, NULL, TASK,
                     THREAD_FLAG_NONE, "kconsole");
-                if (thread != NULL)
-                        thread_ready(thread);
-                else
+                if (thread != NULL) {
+                        thread_start(thread);
+                        thread_detach(thread);
+                } else {
                         log(LF_OTHER, LVL_ERROR,
                             "Unable to create kconsole thread");
+                }
         }
 #endif /* CONFIG_KCONSOLE */
@@ -309 +313 @@
          */
         for (i = 0; i < init.cnt; i++) {
-                if (programs[i].task != NULL)
+                if (programs[i].task != NULL) {
                         program_ready(&programs[i]);
+                        task_release(programs[i].task);
+                }
         }
 

kernel/generic/src/main/main.c

@@ -80 +80 @@
 #include <arch/arch.h>
 #include <arch.h>
-#include <arch/faddr.h>
 #include <ipc/ipc.h>
 #include <macros.h>
@@ -174 +173 @@
             ALIGN_UP((uintptr_t) kdata_end - config.base, PAGE_SIZE);
 
-        context_save(&ctx);
-        context_set(&ctx, FADDR(main_bsp_separated_stack),
+        context_create(&ctx, main_bsp_separated_stack,
             bootstrap_stack, bootstrap_stack_size);
         context_restore(&ctx);
@@ -282 +280 @@
         if (!kinit_thread)
                 panic("Cannot create kinit thread.");
-        thread_ready(kinit_thread);
-
-        /*
-         * This call to scheduler() will return to kinit,
+        thread_start(kinit_thread);
+        thread_detach(kinit_thread);
+
+        /*
+         * This call to scheduler_run() will return to kinit,
          * starting the thread of kernel threads.
          */
-        scheduler();
+        current_copy(CURRENT, (current_t *) CPU_LOCAL->stack);
+        context_replace(scheduler_run, CPU_LOCAL->stack, STACK_SIZE);
         /* not reached */
 }
@@ -328 +328 @@
         ARCH_OP(post_cpu_init);
 
-        current_copy(CURRENT, (current_t *) CPU->stack);
-
         /*
          * If we woke kmp up before we left the kernel stack, we could
@@ -335 +333 @@
          * switch to this cpu's private stack prior to waking kmp up.
          */
-        context_t ctx;
-        context_save(&ctx);
-        context_set(&ctx, FADDR(main_ap_separated_stack),
-            (uintptr_t) CPU->stack, STACK_SIZE);
-        context_restore(&ctx);
+        current_copy(CURRENT, (current_t *) CPU_LOCAL->stack);
+        context_replace(main_ap_separated_stack, CPU_LOCAL->stack, STACK_SIZE);
         /* not reached */
 }
@@ -356 +351 @@
 
         semaphore_up(&ap_completion_semaphore);
-        scheduler();
+        scheduler_run();
         /* not reached */
 }

kernel/generic/src/proc/program.c

@@ -99 +99 @@
         if (!area) {
                 free(kernel_uarg);
-                task_destroy(prg->task);
+                task_release(prg->task);
                 prg->task = NULL;
                 return ENOMEM;
@@ -119 +119 @@
                 free(kernel_uarg);
                 as_area_destroy(as, virt);
-                task_destroy(prg->task);
+                task_release(prg->task);
                 prg->task = NULL;
                 return ELIMIT;
@@ -212 +212 @@
 void program_ready(program_t *prg)
 {
-        thread_ready(prg->main_thread);
+        thread_start(prg->main_thread);
+        thread_detach(prg->main_thread);
         prg->main_thread = NULL;
 }
@@ -251 +252 @@
         program_ready(&prg);
 
+        task_release(prg.task);
+
         return EOK;
 }

kernel/generic/src/proc/scheduler.c

@@ -1 +1 @@
 /*
  * Copyright (c) 2010 Jakub Jermar
+ * Copyright (c) 2023 Jiří Zárevúcky
  * All rights reserved.
  *
@@ -50 +51 @@
 #include <time/delay.h>
 #include <arch/asm.h>
-#include <arch/faddr.h>
 #include <arch/cycle.h>
 #include <atomic.h>
@@ -66 +66 @@
 #include <stacktrace.h>
 
-static void scheduler_separated_stack(void);
-
 atomic_size_t nrdy;  /**< Number of ready threads in the system. */
-
-/** Take actions before new thread runs.
- *
- * Perform actions that need to be
- * taken before the newly selected
- * thread is passed control.
- *
- * THREAD->lock is locked on entry
- *
- */
-static void before_thread_runs(void)
-{
-        before_thread_runs_arch();
-
-#ifdef CONFIG_FPU_LAZY
-        /*
-         * The only concurrent modification possible for fpu_owner here is
-         * another thread changing it from itself to NULL in its destructor.
-         */
-        thread_t *owner = atomic_load_explicit(&CPU->fpu_owner,
-            memory_order_relaxed);
-
-        if (THREAD == owner)
-                fpu_enable();
-        else
-                fpu_disable();
-#elif defined CONFIG_FPU
-        fpu_enable();
-        if (THREAD->fpu_context_exists)
-                fpu_context_restore(&THREAD->fpu_context);
-        else {
-                fpu_init();
-                THREAD->fpu_context_exists = true;
-        }
-#endif
-
-#ifdef CONFIG_UDEBUG
-        if (THREAD->btrace) {
-                istate_t *istate = THREAD->udebug.uspace_state;
-                if (istate != NULL) {
-                        printf("Thread %" PRIu64 " stack trace:\n", THREAD->tid);
-                        stack_trace_istate(istate);
-                }
-
-                THREAD->btrace = false;
-        }
-#endif
-}
-
-/** Take actions after THREAD had run.
- *
- * Perform actions that need to be
- * taken after the running thread
- * had been preempted by the scheduler.
- *
- * THREAD->lock is locked on entry
- *
- */
-static void after_thread_ran(void)
-{
-        after_thread_ran_arch();
-}
 
 #ifdef CONFIG_FPU_LAZY
@@ -207 +143 @@
                 list_remove(&thread->rq_link);
 
-                irq_spinlock_pass(&(CPU->rq[i].lock), &thread->lock);
-
-                thread->cpu = CPU;
-                thread->priority = i;  /* Correct rq index */
-
-                /* Time allocation in microseconds. */
-                uint64_t time_to_run = (i + 1) * 10000;
-
-                /* This is safe because interrupts are disabled. */
-                CPU->preempt_deadline = CPU->current_clock_tick + us2ticks(time_to_run);
-
-                /*
-                 * Clear the stolen flag so that it can be migrated
-                 * when load balancing needs emerge.
-                 */
-                thread->stolen = false;
-                irq_spinlock_unlock(&thread->lock, false);
+                irq_spinlock_unlock(&(CPU->rq[i].lock), false);
 
                 *rq_index = i;
@@ -257 +177 @@
                  * This improves energy saving and hyperthreading.
                  */
-                CPU->idle = true;
+                CPU_LOCAL->idle = true;
 
                 /*
@@ -305 +225 @@
 static void relink_rq(int start)
 {
-        if (CPU->current_clock_tick < CPU->relink_deadline)
+        assert(interrupts_disabled());
+
+        if (CPU_LOCAL->current_clock_tick < CPU_LOCAL->relink_deadline)
                 return;
 
-        CPU->relink_deadline = CPU->current_clock_tick + NEEDS_RELINK_MAX;
+        CPU_LOCAL->relink_deadline = CPU_LOCAL->current_clock_tick + NEEDS_RELINK_MAX;
 
         /* Temporary cache for lists we are moving. */
@@ -340 +262 @@
 }
 
-void scheduler(void)
-{
-        ipl_t ipl = interrupts_disable();
-
-        if (atomic_load(&haltstate))
-                halt();
-
-        if (THREAD) {
-                irq_spinlock_lock(&THREAD->lock, false);
-        }
-
-        scheduler_locked(ipl);
-}
-
-/** The scheduler
- *
- * The thread scheduling procedure.
- * Passes control directly to
- * scheduler_separated_stack().
- *
- */
-void scheduler_locked(ipl_t ipl)
-{
-        assert(CPU != NULL);
-
-        if (THREAD) {
-                /* Update thread kernel accounting */
-                THREAD->kcycles += get_cycle() - THREAD->last_cycle;
-
+/**
+ * Do whatever needs to be done with current FPU state before we switch to
+ * another thread.
+ */
+static void fpu_cleanup(void)
+{
 #if (defined CONFIG_FPU) && (!defined CONFIG_FPU_LAZY)
-                fpu_context_save(&THREAD->fpu_context);
+        fpu_context_save(&THREAD->fpu_context);
 #endif
-                if (!context_save(&THREAD->saved_context)) {
-                        /*
-                         * This is the place where threads leave scheduler();
-                         */
-
-                        /* Save current CPU cycle */
-                        THREAD->last_cycle = get_cycle();
-
-                        irq_spinlock_unlock(&THREAD->lock, false);
-                        interrupts_restore(THREAD->saved_ipl);
-
-                        return;
-                }
-
-                /*
-                 * Interrupt priority level of preempted thread is recorded
-                 * here to facilitate scheduler() invocations from
-                 * interrupts_disable()'d code (e.g. waitq_sleep_timeout()).
-                 *
-                 */
-                THREAD->saved_ipl = ipl;
-        }
-
+}
+
+/**
+ * Set correct FPU state for this thread after switch from another thread.
+ */
+static void fpu_restore(void)
+{
+#ifdef CONFIG_FPU_LAZY
         /*
-         * Through the 'CURRENT' structure, we keep track of THREAD, TASK, CPU, AS
-         * and preemption counter. At this point CURRENT could be coming either
-         * from THREAD's or CPU's stack.
-         *
+         * The only concurrent modification possible for fpu_owner here is
+         * another thread changing it from itself to NULL in its destructor.
          */
-        current_copy(CURRENT, (current_t *) CPU->stack);
+        thread_t *owner = atomic_load_explicit(&CPU->fpu_owner,
+            memory_order_relaxed);
+
+        if (THREAD == owner)
+                fpu_enable();
+        else
+                fpu_disable();
+
+#elif defined CONFIG_FPU
+        fpu_enable();
+        if (THREAD->fpu_context_exists)
+                fpu_context_restore(&THREAD->fpu_context);
+        else {
+                fpu_init();
+                THREAD->fpu_context_exists = true;
+        }
+#endif
+}
+
+/** Things to do before we switch to THREAD context.
+ */
+static void prepare_to_run_thread(int rq_index)
+{
+        relink_rq(rq_index);
+
+        switch_task(THREAD->task);
+
+        assert(atomic_get_unordered(&THREAD->cpu) == CPU);
+
+        atomic_set_unordered(&THREAD->state, Running);
+        atomic_set_unordered(&THREAD->priority, rq_index);  /* Correct rq index */
 
         /*
-         * We may not keep the old stack.
-         * Reason: If we kept the old stack and got blocked, for instance, in
-         * find_best_thread(), the old thread could get rescheduled by another
-         * CPU and overwrite the part of its own stack that was also used by
-         * the scheduler on this CPU.
-         *
-         * Moreover, we have to bypass the compiler-generated POP sequence
-         * which is fooled by SP being set to the very top of the stack.
-         * Therefore the scheduler() function continues in
-         * scheduler_separated_stack().
-         *
+         * Clear the stolen flag so that it can be migrated
+         * when load balancing needs emerge.
          */
-        context_t ctx;
-        context_save(&ctx);
-        context_set(&ctx, FADDR(scheduler_separated_stack),
-            (uintptr_t) CPU->stack, STACK_SIZE);
-        context_restore(&ctx);
-
-        /* Not reached */
-}
-
-/** Scheduler stack switch wrapper
- *
- * Second part of the scheduler() function
- * using new stack. Handling the actual context
- * switch to a new thread.
- *
- */
-void scheduler_separated_stack(void)
-{
-        assert((!THREAD) || (irq_spinlock_locked(&THREAD->lock)));
-        assert(CPU != NULL);
-        assert(interrupts_disabled());
-
-        if (THREAD) {
-                /* Must be run after the switch to scheduler stack */
-                after_thread_ran();
-
-                switch (THREAD->state) {
-                case Running:
-                        irq_spinlock_unlock(&THREAD->lock, false);
-                        thread_ready(THREAD);
-                        break;
-
-                case Exiting:
-                        irq_spinlock_unlock(&THREAD->lock, false);
-                        waitq_close(&THREAD->join_wq);
-
-                        /*
-                         * Release the reference CPU has for the thread.
-                         * If there are no other references (e.g. threads calling join),
-                         * the thread structure is deallocated.
-                         */
-                        thread_put(THREAD);
-                        break;
-
-                case Sleeping:
-                        /*
-                         * Prefer the thread after it's woken up.
-                         */
-                        THREAD->priority = -1;
-                        irq_spinlock_unlock(&THREAD->lock, false);
-                        break;
-
-                default:
-                        /*
-                         * Entering state is unexpected.
-                         */
-                        panic("tid%" PRIu64 ": unexpected state %s.",
-                            THREAD->tid, thread_states[THREAD->state]);
-                        break;
-                }
-
-                THREAD = NULL;
-        }
-
-        int rq_index;
-        THREAD = find_best_thread(&rq_index);
-
-        relink_rq(rq_index);
-
-        switch_task(THREAD->task);
-
-        irq_spinlock_lock(&THREAD->lock, false);
-        THREAD->state = Running;
+        THREAD->stolen = false;
 
 #ifdef SCHEDULER_VERBOSE
         log(LF_OTHER, LVL_DEBUG,
             "cpu%u: tid %" PRIu64 " (priority=%d, ticks=%" PRIu64
-            ", nrdy=%zu)", CPU->id, THREAD->tid, THREAD->priority,
+            ", nrdy=%zu)", CPU->id, THREAD->tid, rq_index,
             THREAD->ticks, atomic_load(&CPU->nrdy));
 #endif
@@ -505 +336 @@
          * function must be executed before the switch to the new stack.
          */
-        before_thread_runs();
+        before_thread_runs_arch();
+
+#ifdef CONFIG_UDEBUG
+        if (atomic_get_unordered(&THREAD->btrace)) {
+                istate_t *istate = THREAD->udebug.uspace_state;
+                if (istate != NULL) {
+                        printf("Thread %" PRIu64 " stack trace:\n", THREAD->tid);
+                        stack_trace_istate(istate);
+                } else {
+                        printf("Thread %" PRIu64 " interrupt state not available\n", THREAD->tid);
+                }
+
+                atomic_set_unordered(&THREAD->btrace, false);
+        }
+#endif
+
+        fpu_restore();
+
+        /* Time allocation in microseconds. */
+        uint64_t time_to_run = (rq_index + 1) * 10000;
+
+        /* Set the time of next preemption. */
+        CPU_LOCAL->preempt_deadline =
+            CPU_LOCAL->current_clock_tick + us2ticks(time_to_run);
+
+        /* Save current CPU cycle */
+        THREAD->last_cycle = get_cycle();
+}
+
+static void add_to_rq(thread_t *thread, cpu_t *cpu, int i)
+{
+        /* Add to the appropriate runqueue. */
+        runq_t *rq = &cpu->rq[i];
+
+        irq_spinlock_lock(&rq->lock, false);
+        list_append(&thread->rq_link, &rq->rq);
+        rq->n++;
+        irq_spinlock_unlock(&rq->lock, false);
+
+        atomic_inc(&nrdy);
+        atomic_inc(&cpu->nrdy);
+}
+
+/** Requeue a thread that was just preempted on this CPU.
+ */
+static void thread_requeue_preempted(thread_t *thread)
+{
+        assert(interrupts_disabled());
+        assert(atomic_get_unordered(&thread->state) == Running);
+        assert(atomic_get_unordered(&thread->cpu) == CPU);
+
+        int prio = atomic_get_unordered(&thread->priority);
+
+        if (prio < RQ_COUNT - 1) {
+                prio++;
+                atomic_set_unordered(&thread->priority, prio);
+        }
+
+        atomic_set_unordered(&thread->state, Ready);
+
+        add_to_rq(thread, CPU, prio);
+}
+
+void thread_requeue_sleeping(thread_t *thread)
+{
+        ipl_t ipl = interrupts_disable();
+
+        assert(atomic_get_unordered(&thread->state) == Sleeping || atomic_get_unordered(&thread->state) == Entering);
+
+        atomic_set_unordered(&thread->priority, 0);
+        atomic_set_unordered(&thread->state, Ready);
+
+        /* Prefer the CPU on which the thread ran last */
+        cpu_t *cpu = atomic_get_unordered(&thread->cpu);
+
+        if (!cpu) {
+                cpu = CPU;
+                atomic_set_unordered(&thread->cpu, CPU);
+        }
+
+        add_to_rq(thread, cpu, 0);
+
+        interrupts_restore(ipl);
+}
+
+static void cleanup_after_thread(thread_t *thread)
+{
+        assert(CURRENT->mutex_locks == 0);
+        assert(interrupts_disabled());
+
+        int expected;
+
+        switch (atomic_get_unordered(&thread->state)) {
+        case Running:
+                thread_requeue_preempted(thread);
+                break;
+
+        case Exiting:
+                waitq_close(&thread->join_wq);
+
+                /*
+                 * Release the reference CPU has for the thread.
+                 * If there are no other references (e.g. threads calling join),
+                 * the thread structure is deallocated.
+                 */
+                thread_put(thread);
+                break;
+
+        case Sleeping:
+                expected = SLEEP_INITIAL;
+
+                /* Only set SLEEP_ASLEEP in sleep pad if it's still in initial state */
+                if (!atomic_compare_exchange_strong_explicit(&thread->sleep_state,
+                    &expected, SLEEP_ASLEEP,
+                    memory_order_acq_rel, memory_order_acquire)) {
+
+                        assert(expected == SLEEP_WOKE);
+                        /* The thread has already been woken up, requeue immediately. */
+                        thread_requeue_sleeping(thread);
+                }
+                break;
+
+        default:
+                /*
+                 * Entering state is unexpected.
+                 */
+                panic("tid%" PRIu64 ": unexpected state %s.",
+                    thread->tid, thread_states[atomic_get_unordered(&thread->state)]);
+                break;
+        }
+}
+
+/** Switch to scheduler context to let other threads run. */
+void scheduler_enter(state_t new_state)
+{
+        ipl_t ipl = interrupts_disable();
+
+        assert(CPU != NULL);
+        assert(THREAD != NULL);
+
+        if (atomic_load(&haltstate))
+                halt();
+
+        /* Check if we have a thread to switch to. */
+
+        int rq_index;
+        thread_t *new_thread = try_find_thread(&rq_index);
+
+        if (new_thread == NULL && new_state == Running) {
+                /* No other thread to run, but we still have work to do here. */
+                interrupts_restore(ipl);
+                return;
+        }
+
+        atomic_set_unordered(&THREAD->state, new_state);
+
+        /* Update thread kernel accounting */
+        atomic_time_increment(&THREAD->kcycles, get_cycle() - THREAD->last_cycle);
+
+        fpu_cleanup();
 
         /*
-         * Copy the knowledge of CPU, TASK, THREAD and preemption counter to
-         * thread's stack.
+         * On Sparc, this saves some extra userspace state that's not
+         * covered by context_save()/context_restore().
          */
-        current_copy(CURRENT, (current_t *) THREAD->kstack);
-
-        context_restore(&THREAD->saved_context);
+        after_thread_ran_arch();
+
+        if (new_thread) {
+                thread_t *old_thread = THREAD;
+                CPU_LOCAL->prev_thread = old_thread;
+                THREAD = new_thread;
+                /* No waiting necessary, we can switch to the new thread directly. */
+                prepare_to_run_thread(rq_index);
+
+                current_copy(CURRENT, (current_t *) new_thread->kstack);
+                context_swap(&old_thread->saved_context, &new_thread->saved_context);
+        } else {
+                /*
+                 * A new thread isn't immediately available, switch to a separate
+                 * stack to sleep or do other idle stuff.
+                 */
+                current_copy(CURRENT, (current_t *) CPU_LOCAL->stack);
+                context_swap(&THREAD->saved_context, &CPU_LOCAL->scheduler_context);
+        }
+
+        assert(CURRENT->mutex_locks == 0);
+        assert(interrupts_disabled());
+
+        /* Check if we need to clean up after another thread. */
+        if (CPU_LOCAL->prev_thread) {
+                cleanup_after_thread(CPU_LOCAL->prev_thread);
+                CPU_LOCAL->prev_thread = NULL;
+        }
+
+        interrupts_restore(ipl);
+}
+
+/** Enter main scheduler loop. Never returns.
+ *
+ * This function switches to a runnable thread as soon as one is available,
+ * after which it is only switched back to if a thread is stopping and there is
+ * no other thread to run in its place. We need a separate context for that
+ * because we're going to block the CPU, which means we need another context
+ * to clean up after the previous thread.
+ */
+void scheduler_run(void)
+{
+        assert(interrupts_disabled());
+
+        assert(CPU != NULL);
+        assert(TASK == NULL);
+        assert(THREAD == NULL);
+        assert(interrupts_disabled());
+
+        while (!atomic_load(&haltstate)) {
+                assert(CURRENT->mutex_locks == 0);
+
+                int rq_index;
+                THREAD = find_best_thread(&rq_index);
+                prepare_to_run_thread(rq_index);
+
+                /*
+                 * Copy the knowledge of CPU, TASK, THREAD and preemption counter to
+                 * thread's stack.
+                 */
+                current_copy(CURRENT, (current_t *) THREAD->kstack);
+
+                /* Switch to thread context. */
+                context_swap(&CPU_LOCAL->scheduler_context, &THREAD->saved_context);
+
+                /* Back from another thread. */
+                assert(CPU != NULL);
+                assert(THREAD != NULL);
+                assert(CURRENT->mutex_locks == 0);
+                assert(interrupts_disabled());
+
+                cleanup_after_thread(THREAD);
+
+                /*
+                 * Necessary because we're allowing interrupts in find_best_thread(),
+                 * so we need to avoid other code referencing the thread we left.
+                 */
+                THREAD = NULL;
+        }
+
+        halt();
+}
+
+/** Thread wrapper.
+ *
+ * This wrapper is provided to ensure that a starting thread properly handles
+ * everything it needs to do when first scheduled, and when it exits.
+ */
+void thread_main_func(void)
+{
+        assert(interrupts_disabled());
+
+        void (*f)(void *) = THREAD->thread_code;
+        void *arg = THREAD->thread_arg;
+
+        /* This is where each thread wakes up after its creation */
+
+        /* Check if we need to clean up after another thread. */
+        if (CPU_LOCAL->prev_thread) {
+                cleanup_after_thread(CPU_LOCAL->prev_thread);
+                CPU_LOCAL->prev_thread = NULL;
+        }
+
+        interrupts_enable();
+
+        f(arg);
+
+        thread_exit();
 
         /* Not reached */
@@ -539 +634 @@
         list_foreach_rev(old_rq->rq, rq_link, thread_t, thread) {
 
-                irq_spinlock_lock(&thread->lock, false);
-
                 /*
                  * Do not steal CPU-wired threads, threads
@@ -547 +640 @@
                  * FPU context is still in the CPU.
                  */
-                if (thread->stolen || thread->nomigrate ||
-                    thread == fpu_owner) {
-                        irq_spinlock_unlock(&thread->lock, false);
+                if (thread->stolen || thread->nomigrate || thread == fpu_owner) {
                         continue;
                 }
 
                 thread->stolen = true;
-                thread->cpu = CPU;
-
-                irq_spinlock_unlock(&thread->lock, false);
+                atomic_set_unordered(&thread->cpu, CPU);
 
                 /*
@@ -659 +748 @@
                  *
                  */
-                scheduler();
+                thread_yield();
         } else {
                 /*
@@ -686 +775 @@
                         continue;
 
-                /* Technically a data race, but we don't really care in this case. */
-                int needs_relink = cpus[cpu].relink_deadline - cpus[cpu].current_clock_tick;
-
-                printf("cpu%u: address=%p, nrdy=%zu, needs_relink=%d\n",
-                    cpus[cpu].id, &cpus[cpu], atomic_load(&cpus[cpu].nrdy),
-                    needs_relink);
+                printf("cpu%u: address=%p, nrdy=%zu\n",
+                    cpus[cpu].id, &cpus[cpu], atomic_load(&cpus[cpu].nrdy));
 
                 unsigned int i;
@@ -705 +790 @@
                             thread) {
                                 printf("%" PRIu64 "(%s) ", thread->tid,
-                                    thread_states[thread->state]);
+                                    thread_states[atomic_get_unordered(&thread->state)]);
                         }
                         printf("\n");

kernel/generic/src/proc/task.c

@@ -158 +158 @@
                 return rc;
 
-        atomic_store(&task->refcount, 0);
         atomic_store(&task->lifecount, 0);
 
@@ -201 +200 @@
         if (!task)
                 return NULL;
+
+        refcount_init(&task->refcount);
 
         task_create_arch(task);
@@ -268 +269 @@
  *
  */
-void task_destroy(task_t *task)
+static void task_destroy(task_t *task)
 {
         /*
@@ -299 +300 @@
 void task_hold(task_t *task)
 {
-        atomic_inc(&task->refcount);
+        refcount_up(&task->refcount);
 }
 
@@ -311 +312 @@
 void task_release(task_t *task)
 {
-        if ((atomic_predec(&task->refcount)) == 0)
+        if (refcount_down(&task->refcount))
                 task_destroy(task);
 }
@@ -416 +417 @@
 /** Find task structure corresponding to task ID.
  *
- * The tasks_lock must be already held by the caller of this function and
- * interrupts must be disabled.
- *
  * @param id Task ID.
  *
- * @return Task structure address or NULL if there is no such task ID.
+ * @return Task reference or NULL if there is no such task ID.
  *
  */
 task_t *task_find_by_id(task_id_t id)
 {
-        assert(interrupts_disabled());
-        assert(irq_spinlock_locked(&tasks_lock));
+        task_t *task = NULL;
+
+        irq_spinlock_lock(&tasks_lock, true);
 
         odlink_t *odlink = odict_find_eq(&tasks, &id, NULL);
-        if (odlink != NULL)
-                return odict_get_instance(odlink, task_t, ltasks);
-
-        return NULL;
+        if (odlink != NULL) {
+                task = odict_get_instance(odlink, task_t, ltasks);
+
+                /*
+                 * The directory of tasks can't hold a reference, since that would
+                 * prevent task from ever being destroyed. That means we have to
+                 * check for the case where the task is already being destroyed, but
+                 * not yet removed from the directory.
+                 */
+                if (!refcount_try_up(&task->refcount))
+                        task = NULL;
+        }
+
+        irq_spinlock_unlock(&tasks_lock, true);
+
+        return task;
 }
 
@@ -506 +517 @@
         /* Current values of threads */
         list_foreach(task->threads, th_link, thread_t, thread) {
-                irq_spinlock_lock(&thread->lock, false);
-
                 /* Process only counted threads */
                 if (!thread->uncounted) {
@@ -515 +524 @@
                         }
 
-                        uret += thread->ucycles;
-                        kret += thread->kcycles;
+                        uret += atomic_time_read(&thread->ucycles);
+                        kret += atomic_time_read(&thread->kcycles);
                 }
-
-                irq_spinlock_unlock(&thread->lock, false);
         }
 
@@ -528 +535 @@
 static void task_kill_internal(task_t *task)
 {
-        irq_spinlock_lock(&task->lock, false);
+        irq_spinlock_lock(&task->lock, true);
 
         /*
@@ -538 +545 @@
         }
 
-        irq_spinlock_unlock(&task->lock, false);
+        irq_spinlock_unlock(&task->lock, true);
 }
 
@@ -556 +563 @@
                 return EPERM;
 
-        irq_spinlock_lock(&tasks_lock, true);
-
         task_t *task = task_find_by_id(id);
-        if (!task) {
-                irq_spinlock_unlock(&tasks_lock, true);
+        if (!task)
                 return ENOENT;
-        }
 
         task_kill_internal(task);
-        irq_spinlock_unlock(&tasks_lock, true);
-
+        task_release(task);
         return EOK;
 }
@@ -596 +598 @@
         }
 
-        irq_spinlock_lock(&tasks_lock, true);
         task_kill_internal(TASK);
-        irq_spinlock_unlock(&tasks_lock, true);
-
         thread_exit();
 }
@@ -628 +627 @@
         if (additional)
                 printf("%-8" PRIu64 " %9zu", task->taskid,
-                    atomic_load(&task->refcount));
+                    atomic_load(&task->lifecount));
         else
                 printf("%-8" PRIu64 " %-14s %-5" PRIu32 " %10p %10p"
@@ -640 +639 @@
                 printf("%-8" PRIu64 " %9" PRIu64 "%c %9" PRIu64 "%c "
                     "%9zu\n", task->taskid, ucycles, usuffix, kcycles,
-                    ksuffix, atomic_load(&task->refcount));
+                    ksuffix, atomic_load(&task->lifecount));
         else
                 printf("%-8" PRIu64 " %-14s %-5" PRIu32 " %18p %18p\n",

kernel/generic/src/proc/thread.c

@@ -60 +60 @@
 #include <arch/interrupt.h>
 #include <smp/ipi.h>
-#include <arch/faddr.h>
 #include <atomic.h>
 #include <memw.h>
@@ -82 +81 @@
 };
 
-enum sleep_state {
-        SLEEP_INITIAL,
-        SLEEP_ASLEEP,
-        SLEEP_WOKE,
-};
-
 /** Lock protecting the @c threads ordered dictionary .
  *
@@ -115 +108 @@
 static int threads_cmp(void *, void *);
 
-/** Thread wrapper.
- *
- * This wrapper is provided to ensure that every thread makes a call to
- * thread_exit() when its implementing function returns.
- *
- * interrupts_disable() is assumed.
- *
- */
-static void cushion(void)
-{
-        void (*f)(void *) = THREAD->thread_code;
-        void *arg = THREAD->thread_arg;
-        THREAD->last_cycle = get_cycle();
-
-        /* This is where each thread wakes up after its creation */
-        irq_spinlock_unlock(&THREAD->lock, false);
-        interrupts_enable();
-
-        f(arg);
-
-        thread_exit();
-
-        /* Not reached */
-}
-
 /** Initialization and allocation for thread_t structure
  *
@@ -147 +115 @@
         thread_t *thread = (thread_t *) obj;
 
-        irq_spinlock_initialize(&thread->lock, "thread_t_lock");
         link_initialize(&thread->rq_link);
         link_initialize(&thread->wq_link);
@@ -229 +196 @@
 void thread_wire(thread_t *thread, cpu_t *cpu)
 {
-        irq_spinlock_lock(&thread->lock, true);
-        thread->cpu = cpu;
+        ipl_t ipl = interrupts_disable();
+        atomic_set_unordered(&thread->cpu, cpu);
         thread->nomigrate++;
-        irq_spinlock_unlock(&thread->lock, true);
-}
-
-/** Invoked right before thread_ready() readies the thread. thread is locked. */
-static void before_thread_is_ready(thread_t *thread)
-{
-        assert(irq_spinlock_locked(&thread->lock));
-}
-
-/** Make thread ready
- *
- * Switch thread to the ready state. Consumes reference passed by the caller.
- *
- * @param thread Thread to make ready.
- *
- */
-void thread_ready(thread_t *thread)
-{
-        irq_spinlock_lock(&thread->lock, true);
-
-        assert(thread->state != Ready);
-
-        before_thread_is_ready(thread);
-
-        int i = (thread->priority < RQ_COUNT - 1) ?
-            ++thread->priority : thread->priority;
-
-        /* Prefer the CPU on which the thread ran last */
-        cpu_t *cpu = thread->cpu ? thread->cpu : CPU;
-
-        thread->state = Ready;
-
-        irq_spinlock_pass(&thread->lock, &(cpu->rq[i].lock));
-
-        /*
-         * Append thread to respective ready queue
-         * on respective processor.
-         */
-
-        list_append(&thread->rq_link, &cpu->rq[i].rq);
-        cpu->rq[i].n++;
-        irq_spinlock_unlock(&(cpu->rq[i].lock), true);
-
-        atomic_inc(&nrdy);
-        atomic_inc(&cpu->nrdy);
+        interrupts_restore(ipl);
+}
+
+/** Start a thread that wasn't started yet since it was created.
+ *
+ * @param thread A reference to the newly created thread.
+ */
+void thread_start(thread_t *thread)
+{
+        assert(atomic_get_unordered(&thread->state) == Entering);
+        thread_requeue_sleeping(thread_ref(thread));
 }
 
@@ -315 +248 @@
         irq_spinlock_unlock(&tidlock, true);
 
-        memset(&thread->saved_context, 0, sizeof(thread->saved_context));
-        context_set(&thread->saved_context, FADDR(cushion),
-            (uintptr_t) thread->kstack, STACK_SIZE);
+        context_create(&thread->saved_context, thread_main_func,
+            thread->kstack, STACK_SIZE);
 
         current_initialize((current_t *) thread->kstack);
-
-        ipl_t ipl = interrupts_disable();
-        thread->saved_ipl = interrupts_read();
-        interrupts_restore(ipl);
 
         str_cpy(thread->name, THREAD_NAME_BUFLEN, name);
@@ -329 +257 @@
         thread->thread_code = func;
         thread->thread_arg = arg;
-        thread->ucycles = 0;
-        thread->kcycles = 0;
+        thread->ucycles = ATOMIC_TIME_INITIALIZER();
+        thread->kcycles = ATOMIC_TIME_INITIALIZER();
         thread->uncounted =
             ((flags & THREAD_FLAG_UNCOUNTED) == THREAD_FLAG_UNCOUNTED);
-        thread->priority = -1;          /* Start in rq[0] */
-        thread->cpu = NULL;
+        atomic_init(&thread->priority, 0);
+        atomic_init(&thread->cpu, NULL);
         thread->stolen = false;
         thread->uspace =
@@ -340 +268 @@
 
         thread->nomigrate = 0;
-        thread->state = Entering;
+        atomic_init(&thread->state, Entering);
 
         atomic_init(&thread->sleep_queue, NULL);
@@ -360 +288 @@
 #ifdef CONFIG_UDEBUG
         /* Initialize debugging stuff */
-        thread->btrace = false;
+        atomic_init(&thread->btrace, false);
         udebug_thread_initialize(&thread->udebug);
 #endif
@@ -404 +332 @@
 
         if (!thread->uncounted) {
-                thread->task->ucycles += thread->ucycles;
-                thread->task->kcycles += thread->kcycles;
+                thread->task->ucycles += atomic_time_read(&thread->ucycles);
+                thread->task->kcycles += atomic_time_read(&thread->kcycles);
         }
 
         irq_spinlock_unlock(&thread->task->lock, false);
 
-        assert((thread->state == Exiting) || (thread->state == Lingering));
+        assert((atomic_get_unordered(&thread->state) == Exiting) || (atomic_get_unordered(&thread->state) == Lingering));
 
         /* Clear cpu->fpu_owner if set to this thread. */
 #ifdef CONFIG_FPU_LAZY
-        if (thread->cpu) {
+        cpu_t *cpu = atomic_get_unordered(&thread->cpu);
+        if (cpu) {
                 /*
                  * We need to lock for this because the old CPU can concurrently try
@@ -420 +349 @@
                  * it to finish. An atomic compare-and-swap wouldn't be enough.
                  */
-                irq_spinlock_lock(&thread->cpu->fpu_lock, false);
-
-                thread_t *owner = atomic_load_explicit(&thread->cpu->fpu_owner,
-                    memory_order_relaxed);
-
-                if (owner == thread) {
-                        atomic_store_explicit(&thread->cpu->fpu_owner, NULL,
-                            memory_order_relaxed);
-                }
-
-                irq_spinlock_unlock(&thread->cpu->fpu_lock, false);
+                irq_spinlock_lock(&cpu->fpu_lock, false);
+
+                if (atomic_get_unordered(&cpu->fpu_owner) == thread)
+                        atomic_set_unordered(&cpu->fpu_owner, NULL);
+
+                irq_spinlock_unlock(&cpu->fpu_lock, false);
         }
 #endif
@@ -525 +449 @@
         }
 
-        irq_spinlock_lock(&THREAD->lock, true);
-        THREAD->state = Exiting;
-        irq_spinlock_unlock(&THREAD->lock, true);
-
-        scheduler();
-
-        panic("should never be reached");
+        scheduler_enter(Exiting);
+        unreachable();
 }
 
@@ -579 +498 @@
 }
 
-static void thread_wait_internal(void)
-{
-        assert(THREAD != NULL);
-
-        ipl_t ipl = interrupts_disable();
-
-        if (atomic_load(&haltstate))
-                halt();
-
-        /*
-         * Lock here to prevent a race between entering the scheduler and another
-         * thread rescheduling this thread.
-         */
-        irq_spinlock_lock(&THREAD->lock, false);
-
-        int expected = SLEEP_INITIAL;
-
-        /* Only set SLEEP_ASLEEP in sleep pad if it's still in initial state */
-        if (atomic_compare_exchange_strong_explicit(&THREAD->sleep_state, &expected,
-            SLEEP_ASLEEP, memory_order_acq_rel, memory_order_acquire)) {
-                THREAD->state = Sleeping;
-                scheduler_locked(ipl);
-        } else {
-                assert(expected == SLEEP_WOKE);
-                /* Return immediately. */
-                irq_spinlock_unlock(&THREAD->lock, false);
-                interrupts_restore(ipl);
-        }
-}
-
 static void thread_wait_timeout_callback(void *arg)
 {
@@ -649 +538 @@
         timeout_t timeout;
 
+        /* Extra check to avoid going to scheduler if we don't need to. */
+        if (atomic_load_explicit(&THREAD->sleep_state, memory_order_acquire) !=
+            SLEEP_INITIAL)
+                return THREAD_WAIT_SUCCESS;
+
         if (deadline != DEADLINE_NEVER) {
-                /* Extra check to avoid setting up a deadline if we don't need to. */
-                if (atomic_load_explicit(&THREAD->sleep_state, memory_order_acquire) !=
-                    SLEEP_INITIAL)
-                        return THREAD_WAIT_SUCCESS;
-
                 timeout_initialize(&timeout);
                 timeout_register_deadline(&timeout, deadline,
@@ -660 +549 @@
         }
 
-        thread_wait_internal();
+        scheduler_enter(Sleeping);
 
         if (deadline != DEADLINE_NEVER && !timeout_unregister(&timeout)) {
@@ -674 +563 @@
 
         int state = atomic_exchange_explicit(&thread->sleep_state, SLEEP_WOKE,
-            memory_order_release);
+            memory_order_acq_rel);
 
         if (state == SLEEP_ASLEEP) {
@@ -682 +571 @@
                  * the waking thread by the sleeper in thread_wait_finish().
                  */
-                thread_ready(thread);
+                thread_requeue_sleeping(thread);
         }
 }
@@ -689 +578 @@
 void thread_migration_disable(void)
 {
+        ipl_t ipl = interrupts_disable();
+
         assert(THREAD);
-
         THREAD->nomigrate++;
+
+        interrupts_restore(ipl);
 }
 
@@ -697 +589 @@
 void thread_migration_enable(void)
 {
+        ipl_t ipl = interrupts_disable();
+
         assert(THREAD);
         assert(THREAD->nomigrate > 0);
@@ -702 +596 @@
         if (THREAD->nomigrate > 0)
                 THREAD->nomigrate--;
+
+        interrupts_restore(ipl);
 }
 
@@ -731 +627 @@
 
 /** Wait for another thread to exit.
- * This function does not destroy the thread. Reference counting handles that.
+ * After successful wait, the thread reference is destroyed.
  *
  * @param thread Thread to join on exit.
@@ -742 +638 @@
 errno_t thread_join_timeout(thread_t *thread, uint32_t usec, unsigned int flags)
 {
+        assert(thread != NULL);
+
         if (thread == THREAD)
                 return EINVAL;
 
-        irq_spinlock_lock(&thread->lock, true);
-        state_t state = thread->state;
-        irq_spinlock_unlock(&thread->lock, true);
-
-        if (state == Exiting) {
-                return EOK;
-        } else {
-                return _waitq_sleep_timeout(&thread->join_wq, usec, flags);
-        }
+        errno_t rc = _waitq_sleep_timeout(&thread->join_wq, usec, flags);
+
+        if (rc == EOK)
+                thread_put(thread);
+
+        return rc;
+}
+
+void thread_detach(thread_t *thread)
+{
+        thread_put(thread);
 }
 
@@ -770 +670 @@
 
         (void) waitq_sleep_timeout(&wq, usec);
+}
+
+/** Allow other threads to run. */
+void thread_yield(void)
+{
+        assert(THREAD != NULL);
+        scheduler_enter(Running);
 }
 
@@ -776 +683 @@
         uint64_t ucycles, kcycles;
         char usuffix, ksuffix;
-        order_suffix(thread->ucycles, &ucycles, &usuffix);
-        order_suffix(thread->kcycles, &kcycles, &ksuffix);
+        order_suffix(atomic_time_read(&thread->ucycles), &ucycles, &usuffix);
+        order_suffix(atomic_time_read(&thread->kcycles), &kcycles, &ksuffix);
+
+        state_t state = atomic_get_unordered(&thread->state);
 
         char *name;
@@ -791 +700 @@
         else
                 printf("%-8" PRIu64 " %-14s %p %-8s %p %-5" PRIu32 "\n",
-                    thread->tid, name, thread, thread_states[thread->state],
+                    thread->tid, name, thread, thread_states[state],
                     thread->task, thread->task->container);
 
         if (additional) {
-                if (thread->cpu)
-                        printf("%-5u", thread->cpu->id);
+                cpu_t *cpu = atomic_get_unordered(&thread->cpu);
+                if (cpu)
+                        printf("%-5u", cpu->id);
                 else
                         printf("none ");
 
-                if (thread->state == Sleeping) {
+                if (state == Sleeping) {
                         printf(" %p", thread->sleep_queue);
                 }
@@ -879 +789 @@
 void thread_update_accounting(bool user)
 {
+        assert(interrupts_disabled());
+
         uint64_t time = get_cycle();
 
-        assert(interrupts_disabled());
-        assert(irq_spinlock_locked(&THREAD->lock));
-
         if (user)
-                THREAD->ucycles += time - THREAD->last_cycle;
+                atomic_time_increment(&THREAD->ucycles, time - THREAD->last_cycle);
         else
-                THREAD->kcycles += time - THREAD->last_cycle;
+                atomic_time_increment(&THREAD->kcycles, time - THREAD->last_cycle);
 
         THREAD->last_cycle = time;
@@ -998 +907 @@
          */
 
-        irq_spinlock_lock(&thread->lock, true);
-
-        bool sleeping = false;
-        istate_t *istate = thread->udebug.uspace_state;
-        if (istate != NULL) {
-                printf("Scheduling thread stack trace.\n");
-                thread->btrace = true;
-                if (thread->state == Sleeping)
-                        sleeping = true;
-        } else
-                printf("Thread interrupt state not available.\n");
-
-        irq_spinlock_unlock(&thread->lock, true);
-
-        if (sleeping)
-                thread_wakeup(thread);
-
+        printf("Scheduling thread stack trace.\n");
+        atomic_set_unordered(&thread->btrace, true);
+
+        thread_wakeup(thread);
         thread_put(thread);
 }
@@ -1115 +1011 @@
                 thread_attach(thread, TASK);
 #endif
-                thread_ready(thread);
+                thread_start(thread);
+                thread_put(thread);
 
                 return 0;

kernel/generic/src/security/perm.c

@@ -89 +89 @@
                 return EPERM;
 
-        irq_spinlock_lock(&tasks_lock, true);
         task_t *task = task_find_by_id(taskid);
-
-        if ((!task) || (!container_check(CONTAINER, task->container))) {
-                irq_spinlock_unlock(&tasks_lock, true);
+        if (!task)
                 return ENOENT;
+
+        errno_t rc = ENOENT;
+
+        irq_spinlock_lock(&task->lock, true);
+        if (container_check(CONTAINER, task->container)) {
+                task->perms |= perms;
+                rc = EOK;
         }
-
-        irq_spinlock_lock(&task->lock, false);
-        task->perms |= perms;
-        irq_spinlock_unlock(&task->lock, false);
-
-        irq_spinlock_unlock(&tasks_lock, true);
-        return EOK;
+        irq_spinlock_unlock(&task->lock, true);
+
+        task_release(task);
+        return rc;
 }
 
@@ -118 +119 @@
 static errno_t perm_revoke(task_id_t taskid, perm_t perms)
 {
-        irq_spinlock_lock(&tasks_lock, true);
-
         task_t *task = task_find_by_id(taskid);
-        if ((!task) || (!container_check(CONTAINER, task->container))) {
-                irq_spinlock_unlock(&tasks_lock, true);
+        if (!task)
                 return ENOENT;
-        }
 
         /*
@@ -131 +128 @@
          * doesn't have PERM_PERM.
          */
-        irq_spinlock_lock(&TASK->lock, false);
-
-        if ((!(TASK->perms & PERM_PERM)) || (task != TASK)) {
-                irq_spinlock_unlock(&TASK->lock, false);
-                irq_spinlock_unlock(&tasks_lock, true);
+        if (task != TASK && !(perm_get(TASK) & PERM_PERM)) {
+                task_release(task);
                 return EPERM;
         }
 
-        task->perms &= ~perms;
-        irq_spinlock_unlock(&TASK->lock, false);
-
-        irq_spinlock_unlock(&tasks_lock, true);
-        return EOK;
+        errno_t rc = ENOENT;
+
+        irq_spinlock_lock(&task->lock, true);
+        if (container_check(CONTAINER, task->container)) {
+                task->perms &= ~perms;
+                rc = EOK;
+        }
+        irq_spinlock_unlock(&task->lock, true);
+
+        task_release(task);
+        return rc;
 }
 

kernel/generic/src/syscall/syscall.c

@@ -141 +141 @@
 {
         /* Do userpace accounting */
-        irq_spinlock_lock(&THREAD->lock, true);
+        ipl_t ipl = interrupts_disable();
         thread_update_accounting(true);
-        irq_spinlock_unlock(&THREAD->lock, true);
+        interrupts_restore(ipl);
 
 #ifdef CONFIG_UDEBUG
@@ -191 +191 @@
 
         /* Do kernel accounting */
-        irq_spinlock_lock(&THREAD->lock, true);
+        ipl = interrupts_disable();
         thread_update_accounting(false);
-        irq_spinlock_unlock(&THREAD->lock, true);
+        interrupts_restore(ipl);
 
         return rc;

kernel/generic/src/sysinfo/stats.c

@@ -221 +221 @@
         stats_task->virtmem = get_task_virtmem(task->as);
         stats_task->resmem = get_task_resmem(task->as);
-        stats_task->threads = atomic_load(&task->refcount);
+        stats_task->threads = atomic_load(&task->lifecount);
         task_get_accounting(task, &(stats_task->ucycles),
             &(stats_task->kcycles));
@@ -299 +299 @@
 {
         assert(interrupts_disabled());
-        assert(irq_spinlock_locked(&thread->lock));
 
         stats_thread->thread_id = thread->tid;
         stats_thread->task_id = thread->task->taskid;
-        stats_thread->state = thread->state;
-        stats_thread->priority = thread->priority;
-        stats_thread->ucycles = thread->ucycles;
-        stats_thread->kcycles = thread->kcycles;
-
-        if (thread->cpu != NULL) {
+        stats_thread->state = atomic_get_unordered(&thread->state);
+        stats_thread->priority = atomic_get_unordered(&thread->priority);
+        stats_thread->ucycles = atomic_time_read(&thread->ucycles);
+        stats_thread->kcycles = atomic_time_read(&thread->kcycles);
+
+        cpu_t *cpu = atomic_get_unordered(&thread->cpu);
+
+        if (cpu != NULL) {
                 stats_thread->on_cpu = true;
-                stats_thread->cpu = thread->cpu->id;
+                stats_thread->cpu = cpu->id;
         } else
                 stats_thread->on_cpu = false;
@@ -361 +362 @@
         thread_t *thread = thread_first();
         while (thread != NULL) {
-                /* Interrupts are already disabled */
-                irq_spinlock_lock(&thread->lock, false);
-
                 /* Record the statistics and increment the index */
                 produce_stats_thread(thread, &stats_threads[i]);
                 i++;
-
-                irq_spinlock_unlock(&thread->lock, false);
 
                 thread = thread_next(thread);
@@ -515 +511 @@
 {
         /* Initially no return value */
-        sysinfo_return_t ret;
-        ret.tag = SYSINFO_VAL_UNDEFINED;
+        sysinfo_return_t ret = {
+                .tag = SYSINFO_VAL_UNDEFINED,
+        };
 
         /* Parse the task ID */
@@ -523 +520 @@
                 return ret;
 
-        /* Messing with task structures, avoid deadlock */
-        irq_spinlock_lock(&tasks_lock, true);
-
         task_t *task = task_find_by_id(task_id);
-        if (task == NULL) {
-                /* No task with this ID */
-                irq_spinlock_unlock(&tasks_lock, true);
+        if (!task)
                 return ret;
-        }
 
         if (dry_run) {
@@ -537 +528 @@
                 ret.data.data = NULL;
                 ret.data.size = sizeof(stats_task_t);
-
-                irq_spinlock_unlock(&tasks_lock, true);
         } else {
                 /* Allocate stats_task_t structure */
-                stats_task_t *stats_task =
-                    (stats_task_t *) malloc(sizeof(stats_task_t));
-                if (stats_task == NULL) {
-                        irq_spinlock_unlock(&tasks_lock, true);
-                        return ret;
+                stats_task_t *stats_task = malloc(sizeof(stats_task_t));
+
+                if (stats_task != NULL) {
+                        /* Correct return value */
+                        ret.tag = SYSINFO_VAL_FUNCTION_DATA;
+                        ret.data.data = stats_task;
+                        ret.data.size = sizeof(stats_task_t);
+
+                        irq_spinlock_lock(&task->lock, true);
+                        produce_stats_task(task, stats_task);
+                        irq_spinlock_unlock(&task->lock, true);
                 }
-
-                /* Correct return value */
-                ret.tag = SYSINFO_VAL_FUNCTION_DATA;
-                ret.data.data = (void *) stats_task;
-                ret.data.size = sizeof(stats_task_t);
-
-                /* Hand-over-hand locking */
-                irq_spinlock_exchange(&tasks_lock, &task->lock);
-
-                produce_stats_task(task, stats_task);
-
-                irq_spinlock_unlock(&task->lock, true);
-        }
-
+        }
+
+        task_release(task);
         return ret;
 }
@@ -624 +608 @@
                 ret.data.size = sizeof(stats_thread_t);
 
-                /*
-                 * Replaced hand-over-hand locking with regular nested sections
-                 * to avoid weak reference leak issues.
-                 */
-                irq_spinlock_lock(&thread->lock, false);
                 produce_stats_thread(thread, stats_thread);
-                irq_spinlock_unlock(&thread->lock, false);
 
                 irq_spinlock_unlock(&threads_lock, true);

kernel/generic/src/time/clock.c

@@ -123 +123 @@
 static void cpu_update_accounting(void)
 {
+        // FIXME: get_cycle() is unimplemented on several platforms
         uint64_t now = get_cycle();
-        atomic_time_increment(&CPU->busy_cycles, now - CPU->last_cycle);
-        CPU->last_cycle = now;
+        atomic_time_increment(&CPU->busy_cycles, now - CPU_LOCAL->last_cycle);
+        CPU_LOCAL->last_cycle = now;
 }
 
@@ -137 +138 @@
 void clock(void)
 {
-        size_t missed_clock_ticks = CPU->missed_clock_ticks;
-        CPU->missed_clock_ticks = 0;
-
-        CPU->current_clock_tick += missed_clock_ticks + 1;
-        uint64_t current_clock_tick = CPU->current_clock_tick;
+        size_t missed_clock_ticks = CPU_LOCAL->missed_clock_ticks;
+        CPU_LOCAL->missed_clock_ticks = 0;
+
+        CPU_LOCAL->current_clock_tick += missed_clock_ticks + 1;
+        uint64_t current_clock_tick = CPU_LOCAL->current_clock_tick;
         clock_update_counters(current_clock_tick);
 
@@ -186 +187 @@
 
         if (THREAD) {
-                if (current_clock_tick >= CPU->preempt_deadline && PREEMPTION_ENABLED) {
-                        scheduler();
+                if (current_clock_tick >= CPU_LOCAL->preempt_deadline && PREEMPTION_ENABLED) {
+                        thread_yield();
 #ifdef CONFIG_UDEBUG
                         /*

kernel/generic/src/time/timeout.c

@@ -77 +77 @@
                 return 0;
 
-        return CPU->current_clock_tick + us2ticks(usec);
+        return CPU_LOCAL->current_clock_tick + us2ticks(usec);
 }
 

kernel/generic/src/udebug/udebug_ops.c

@@ -90 +90 @@
         }
 
-        irq_spinlock_lock(&thread->lock, true);
-
         /* Verify that 'thread' is a userspace thread. */
         if (!thread->uspace) {
-                /* It's not, deny its existence */
-                irq_spinlock_unlock(&thread->lock, true);
                 mutex_unlock(&TASK->udebug.lock);
                 return ENOENT;
         }
-
-        /* Verify debugging state. */
-        if (thread->udebug.active != true) {
-                /* Not in debugging session or undesired GO state */
-                irq_spinlock_unlock(&thread->lock, true);
-                mutex_unlock(&TASK->udebug.lock);
-                return ENOENT;
-        }
-
-        /* Now verify that the thread belongs to the current task. */
-        if (thread->task != TASK) {
-                /* No such thread belonging this task */
-                irq_spinlock_unlock(&thread->lock, true);
-                mutex_unlock(&TASK->udebug.lock);
-                return ENOENT;
-        }
-
-        irq_spinlock_unlock(&thread->lock, true);
-
-        /* Only mutex TASK->udebug.lock left. */
 
         /*
@@ -126 +102 @@
          */
         mutex_lock(&thread->udebug.lock);
+
+        /* Verify debugging state. */
+        if (thread->udebug.active != true) {
+                /* Not in debugging session or undesired GO state */
+                mutex_unlock(&thread->udebug.lock);
+                mutex_unlock(&TASK->udebug.lock);
+                return ENOENT;
+        }
+
+        /* Now verify that the thread belongs to the current task. */
+        if (thread->task != TASK) {
+                /* No such thread belonging this task */
+                mutex_unlock(&thread->udebug.lock);
+                mutex_unlock(&TASK->udebug.lock);
+                return ENOENT;
+        }
 
         /* The big task mutex is no longer needed. */
@@ -388 +380 @@
         /* FIXME: make sure the thread isn't past debug shutdown... */
         list_foreach(TASK->threads, th_link, thread_t, thread) {
-                irq_spinlock_lock(&thread->lock, false);
                 bool uspace = thread->uspace;
-                irq_spinlock_unlock(&thread->lock, false);
 
                 /* Not interested in kernel threads. */

kernel/test/mm/falloc2.c

@@ -43 +43 @@
 #define THREADS      8
 
-static atomic_size_t thread_cnt;
 static atomic_size_t thread_fail;
 
@@ -56 +55 @@
                     "Unable to allocate frames\n", THREAD->tid, CPU->id);
                 atomic_inc(&thread_fail);
-                atomic_dec(&thread_cnt);
                 return;
         }
@@ -108 +106 @@
         TPRINTF("Thread #%" PRIu64 " (cpu%u): Exiting\n",
             THREAD->tid, CPU->id);
-        atomic_dec(&thread_cnt);
 }
 
 const char *test_falloc2(void)
 {
-        atomic_store(&thread_cnt, THREADS);
         atomic_store(&thread_fail, 0);
+
+        thread_t *threads[THREADS] = { };
 
         for (unsigned int i = 0; i < THREADS; i++) {
@@ -123 +121 @@
                         break;
                 }
-                thread_ready(thrd);
+                thread_start(thrd);
+                threads[i] = thrd;
         }
 
-        while (atomic_load(&thread_cnt) > 0) {
-                TPRINTF("Threads left: %zu\n",
-                    atomic_load(&thread_cnt));
-                thread_sleep(1);
+        for (unsigned int i = 0; i < THREADS; i++) {
+                if (threads[i] != NULL)
+                        thread_join(threads[i]);
+
+                TPRINTF("Threads left: %u\n", THREADS - i - 1);
         }
 

kernel/test/mm/slab1.c

@@ -121 +121 @@
 static void *thr_data[THREADS][THR_MEM_COUNT];
 static slab_cache_t *thr_cache;
-static semaphore_t thr_sem;
 
 static void slabtest(void *data)
@@ -142 +141 @@
 
         TPRINTF("Thread #%" PRIu64 " finished\n", THREAD->tid);
-
-        semaphore_up(&thr_sem);
 }
 
 static void testthreads(void)
 {
-        thread_t *t;
-        int i;
-
         thr_cache = slab_cache_create("thread_cache", THR_MEM_SIZE, 0, NULL, NULL,
             SLAB_CACHE_NOMAGAZINE);
 
-        semaphore_initialize(&thr_sem, 0);
-        for (i = 0; i < THREADS; i++) {
-                if (!(t = thread_create(slabtest, (void *) (sysarg_t) i, TASK, THREAD_FLAG_NONE, "slabtest"))) {
+        thread_t *threads[THREADS] = { };
+
+        for (int i = 0; i < THREADS; i++) {
+                threads[i] = thread_create(slabtest, (void *) (sysarg_t) i,
+                    TASK, THREAD_FLAG_NONE, "slabtest");
+                if (threads[i]) {
+                        thread_start(threads[i]);
+                } else {
                         TPRINTF("Could not create thread %d\n", i);
-                } else
-                        thread_ready(t);
+                }
         }
 
-        for (i = 0; i < THREADS; i++)
-                semaphore_down(&thr_sem);
+        for (int i = 0; i < THREADS; i++) {
+                if (threads[i] != NULL)
+                        thread_join(threads[i]);
+        }
 
         slab_cache_destroy(thr_cache);

kernel/test/mm/slab2.c

@@ -127 +127 @@
 
 static slab_cache_t *thr_cache;
-static semaphore_t thr_sem;
 static condvar_t thread_starter;
 static mutex_t starter_mutex;
@@ -188 +187 @@
         if (!test_quiet)
                 slab_print_list();
-
-        semaphore_up(&thr_sem);
 }
 
@@ -198 +195 @@
          * then release everything, then again allocate, then release
          */
-        thread_t *t;
-        int i;
 
         TPRINTF("Running stress test with size %d\n", size);
@@ -207 +202 @@
 
         thr_cache = slab_cache_create("thread_cache", size, 0, NULL, NULL, 0);
-        semaphore_initialize(&thr_sem, 0);
-        for (i = 0; i < THREADS; i++) {
-                if (!(t = thread_create(slabtest, NULL, TASK, THREAD_FLAG_NONE, "slabtest"))) {
+
+        thread_t *threads[THREADS] = { };
+
+        for (int i = 0; i < THREADS; i++) {
+                threads[i] = thread_create(slabtest, NULL,
+                    TASK, THREAD_FLAG_NONE, "slabtest");
+                if (threads[i]) {
+                        thread_start(threads[i]);
+                } else {
                         TPRINTF("Could not create thread %d\n", i);
-                } else
-                        thread_ready(t);
-        }
+                }
+        }
+
         thread_sleep(1);
         condvar_broadcast(&thread_starter);
 
-        for (i = 0; i < THREADS; i++)
-                semaphore_down(&thr_sem);
+        for (int i = 0; i < THREADS; i++) {
+                if (threads[i] != NULL)
+                        thread_join(threads[i]);
+        }
 
         slab_cache_destroy(thr_cache);

kernel/test/synch/semaphore1.c

@@ -89 +89 @@
                                 thrd = thread_create(consumer, NULL, TASK,
                                     THREAD_FLAG_NONE, "consumer");
-                                if (thrd)
-                                        thread_ready(thrd);
-                                else
+                                if (thrd) {
+                                        thread_start(thrd);
+                                        thread_detach(thrd);
+                                } else {
                                         TPRINTF("could not create consumer %d\n", i);
+                                }
                         }
                         for (k = 0; k < (4 - i); k++) {
                                 thrd = thread_create(producer, NULL, TASK,
                                     THREAD_FLAG_NONE, "producer");
-                                if (thrd)
-                                        thread_ready(thrd);
-                                else
+                                if (thrd) {
+                                        thread_start(thrd);
+                                        thread_detach(thrd);
+                                } else {
                                         TPRINTF("could not create producer %d\n", i);
+                                }
                         }
                 }

kernel/test/synch/semaphore2.c

@@ -92 +92 @@
                 thrd = thread_create(consumer, NULL, TASK,
                     THREAD_FLAG_NONE, "consumer");
-                if (thrd)
-                        thread_ready(thrd);
-                else
+                if (thrd) {
+                        thread_start(thrd);
+                        thread_detach(thrd);
+                } else {
                         TPRINTF("Error creating thread\n");
+                }
         }
 

kernel/test/thread/thread1.c

@@ -38 +38 @@
 
 static atomic_bool finish;
-static atomic_size_t threads_finished;
 
 static void threadtest(void *data)
@@ -46 +45 @@
                 thread_usleep(100000);
         }
-        atomic_inc(&threads_finished);
 }
 
 const char *test_thread1(void)
 {
-        unsigned int i;
-        size_t total = 0;
+        atomic_store(&finish, true);
 
-        atomic_store(&finish, true);
-        atomic_store(&threads_finished, 0);
+        thread_t *threads[THREADS] = { };
 
-        for (i = 0; i < THREADS; i++) {
-                thread_t *t;
-                if (!(t = thread_create(threadtest, NULL, TASK,
-                    THREAD_FLAG_NONE, "threadtest"))) {
+        for (int i = 0; i < THREADS; i++) {
+                threads[i] = thread_create(threadtest, NULL,
+                    TASK, THREAD_FLAG_NONE, "threadtest");
+
+                if (threads[i]) {
+                        thread_start(threads[i]);
+                } else {
                         TPRINTF("Could not create thread %d\n", i);
                         break;
                 }
-                thread_ready(t);
-                total++;
         }
 
@@ -72 +69 @@
 
         atomic_store(&finish, false);
-        while (atomic_load(&threads_finished) < total) {
-                TPRINTF("Threads left: %zu\n", total - atomic_load(&threads_finished));
-                thread_sleep(1);
+
+        for (int i = 0; i < THREADS; i++) {
+                if (threads[i] != NULL)
+                        thread_join(threads[i]);
+
+                TPRINTF("Threads left: %d\n", THREADS - i - 1);
         }