Index: kernel/arch/abs32le/include/asm.h =================================================================== --- kernel/arch/abs32le/include/asm.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/abs32le/include/asm.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -38,10 +38,11 @@ #include #include - -static inline void asm_delay_loop(uint32_t usec) -{ -} - -static inline __attribute__((noreturn)) void cpu_halt(void) +#include + +NO_TRACE static inline void asm_delay_loop(uint32_t usec) +{ +} + +NO_TRACE static inline __attribute__((noreturn)) void cpu_halt(void) { /* On real hardware this should stop processing further @@ -53,5 +54,5 @@ } -static inline void cpu_sleep(void) +NO_TRACE static inline void cpu_sleep(void) { /* On real hardware this should put the CPU into low-power @@ -61,5 +62,5 @@ } -static inline void pio_write_8(ioport8_t *port, uint8_t val) +NO_TRACE static inline void pio_write_8(ioport8_t *port, uint8_t val) { } @@ -73,5 +74,5 @@ * */ -static inline void pio_write_16(ioport16_t *port, uint16_t val) +NO_TRACE static inline void pio_write_16(ioport16_t *port, uint16_t val) { } @@ -85,5 +86,5 @@ * */ -static inline void pio_write_32(ioport32_t *port, uint32_t val) +NO_TRACE static inline void pio_write_32(ioport32_t *port, uint32_t val) { } @@ -97,5 +98,5 @@ * */ -static inline uint8_t pio_read_8(ioport8_t *port) +NO_TRACE static inline uint8_t pio_read_8(ioport8_t *port) { return 0; @@ -110,5 +111,5 @@ * */ -static inline uint16_t pio_read_16(ioport16_t *port) +NO_TRACE static inline uint16_t pio_read_16(ioport16_t *port) { return 0; @@ -123,59 +124,72 @@ * */ -static inline uint32_t pio_read_32(ioport32_t *port) -{ - return 0; -} - -static inline ipl_t interrupts_enable(void) -{ - /* On real hardware this unconditionally enables preemption - by internal and external interrupts. - - The return value stores the previous interrupt level. */ - - return 0; -} - -static inline ipl_t interrupts_disable(void) -{ - /* On real hardware this disables preemption by the usual - set of internal and external interrupts. This does not - apply to special non-maskable interrupts and sychronous - CPU exceptions. - - The return value stores the previous interrupt level. */ - - return 0; -} - -static inline void interrupts_restore(ipl_t ipl) -{ - /* On real hardware this either enables or disables preemption - according to the interrupt level value from the argument. */ -} - -static inline ipl_t interrupts_read(void) -{ - /* On real hardware the return value stores the current interrupt - level. */ - - return 0; -} - -static inline bool interrupts_disabled(void) -{ - /* On real hardware the return value is true iff interrupts are - disabled. */ +NO_TRACE static inline uint32_t pio_read_32(ioport32_t *port) +{ + return 0; +} + +NO_TRACE static inline ipl_t interrupts_enable(void) +{ + /* + * On real hardware this unconditionally enables preemption + * by internal and external interrupts. + * + * The return value stores the previous interrupt level. + */ + + return 0; +} + +NO_TRACE static inline ipl_t interrupts_disable(void) +{ + /* + * On real hardware this disables preemption by the usual + * set of internal and external interrupts. This does not + * apply to special non-maskable interrupts and sychronous + * CPU exceptions. + * + * The return value stores the previous interrupt level. + */ + + return 0; +} + +NO_TRACE static inline void interrupts_restore(ipl_t ipl) +{ + /* + * On real hardware this either enables or disables preemption + * according to the interrupt level value from the argument. + */ +} + +NO_TRACE static inline ipl_t interrupts_read(void) +{ + /* + * On real hardware the return value stores the current interrupt + * level. + */ + + return 0; +} + +NO_TRACE static inline bool interrupts_disabled(void) +{ + /* + * On real hardware the return value is true iff interrupts are + * disabled. + */ + return false; } -static inline uintptr_t get_stack_base(void) -{ - /* On real hardware this returns the address of the bottom - of the current CPU stack. The the_t structure is stored - on the bottom of stack and this is used to identify the - current CPU, current task, current thread and current - address space. */ +NO_TRACE static inline uintptr_t get_stack_base(void) +{ + /* + * On real hardware this returns the address of the bottom + * of the current CPU stack. The the_t structure is stored + * on the bottom of stack and this is used to identify the + * current CPU, current task, current thread and current + * address space. + */ return 0; Index: kernel/arch/abs32le/include/atomic.h =================================================================== --- kernel/arch/abs32le/include/atomic.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/abs32le/include/atomic.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -40,6 +40,7 @@ #include #include +#include -ATOMIC static inline void atomic_inc(atomic_t *val) +NO_TRACE ATOMIC static inline void atomic_inc(atomic_t *val) WRITES(&val->count) REQUIRES_EXTENT_MUTABLE(val) @@ -52,5 +53,5 @@ } -ATOMIC static inline void atomic_dec(atomic_t *val) +NO_TRACE ATOMIC static inline void atomic_dec(atomic_t *val) WRITES(&val->count) REQUIRES_EXTENT_MUTABLE(val) @@ -63,5 +64,5 @@ } -ATOMIC static inline atomic_count_t atomic_postinc(atomic_t *val) +NO_TRACE ATOMIC static inline atomic_count_t atomic_postinc(atomic_t *val) WRITES(&val->count) REQUIRES_EXTENT_MUTABLE(val) @@ -78,5 +79,5 @@ } -ATOMIC static inline atomic_count_t atomic_postdec(atomic_t *val) +NO_TRACE ATOMIC static inline atomic_count_t atomic_postdec(atomic_t *val) WRITES(&val->count) REQUIRES_EXTENT_MUTABLE(val) @@ -96,5 +97,5 @@ #define atomic_predec(val) (atomic_postdec(val) - 1) -ATOMIC static inline atomic_count_t test_and_set(atomic_t *val) +NO_TRACE ATOMIC static inline atomic_count_t test_and_set(atomic_t *val) WRITES(&val->count) REQUIRES_EXTENT_MUTABLE(val) @@ -109,5 +110,5 @@ } -static inline void atomic_lock_arch(atomic_t *val) +NO_TRACE static inline void atomic_lock_arch(atomic_t *val) WRITES(&val->count) REQUIRES_EXTENT_MUTABLE(val) Index: kernel/arch/abs32le/include/cycle.h =================================================================== --- kernel/arch/abs32le/include/cycle.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/abs32le/include/cycle.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -36,5 +36,7 @@ #define KERN_abs32le_CYCLE_H_ -static inline uint64_t get_cycle(void) +#include + +NO_TRACE static inline uint64_t get_cycle(void) { return 0; Index: kernel/arch/abs32le/include/interrupt.h =================================================================== --- kernel/arch/abs32le/include/interrupt.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/abs32le/include/interrupt.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -38,4 +38,5 @@ #include #include +#include #define IVT_ITEMS 0 @@ -54,5 +55,5 @@ } istate_t; -static inline int istate_from_uspace(istate_t *istate) +NO_TRACE static inline int istate_from_uspace(istate_t *istate) REQUIRES_EXTENT_MUTABLE(istate) { @@ -63,5 +64,6 @@ } -static inline void istate_set_retaddr(istate_t *istate, uintptr_t retaddr) +NO_TRACE static inline void istate_set_retaddr(istate_t *istate, + uintptr_t retaddr) WRITES(&istate->ip) { @@ -71,5 +73,5 @@ } -static inline unative_t istate_get_pc(istate_t *istate) +NO_TRACE static inline unative_t istate_get_pc(istate_t *istate) REQUIRES_EXTENT_MUTABLE(istate) { @@ -79,5 +81,5 @@ } -static inline unative_t istate_get_fp(istate_t *istate) +NO_TRACE static inline unative_t istate_get_fp(istate_t *istate) REQUIRES_EXTENT_MUTABLE(istate) { Index: kernel/arch/abs32le/include/mm/page.h =================================================================== --- kernel/arch/abs32le/include/mm/page.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/abs32le/include/mm/page.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -37,4 +37,5 @@ #include +#include #define PAGE_WIDTH FRAME_WIDTH @@ -139,5 +140,5 @@ } __attribute__((packed)) pte_t; -static inline unsigned int get_pt_flags(pte_t *pt, size_t i) +NO_TRACE static inline unsigned int get_pt_flags(pte_t *pt, size_t i) REQUIRES_ARRAY_MUTABLE(pt, PTL0_ENTRIES_ARCH) { @@ -155,5 +156,5 @@ } -static inline void set_pt_flags(pte_t *pt, size_t i, int flags) +NO_TRACE static inline void set_pt_flags(pte_t *pt, size_t i, int flags) WRITES(ARRAY_RANGE(pt, PTL0_ENTRIES_ARCH)) REQUIRES_ARRAY_MUTABLE(pt, PTL0_ENTRIES_ARCH) Index: kernel/arch/amd64/include/asm.h =================================================================== --- kernel/arch/amd64/include/asm.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/amd64/include/asm.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -39,7 +39,5 @@ #include #include - -extern void asm_delay_loop(uint32_t t); -extern void asm_fake_loop(uint32_t t); +#include /** Return base address of current stack. @@ -50,5 +48,5 @@ * */ -static inline uintptr_t get_stack_base(void) +NO_TRACE static inline uintptr_t get_stack_base(void) { uintptr_t v; @@ -57,5 +55,5 @@ "andq %%rsp, %[v]\n" : [v] "=r" (v) - : "0" (~((uint64_t) STACK_SIZE-1)) + : "0" (~((uint64_t) STACK_SIZE - 1)) ); @@ -63,10 +61,12 @@ } -static inline void cpu_sleep(void) -{ - asm volatile ("hlt\n"); -} - -static inline void __attribute__((noreturn)) cpu_halt(void) +NO_TRACE static inline void cpu_sleep(void) +{ + asm volatile ( + "hlt\n" + ); +} + +NO_TRACE static inline void __attribute__((noreturn)) cpu_halt(void) { while (true) { @@ -77,5 +77,4 @@ } - /** Byte from port * @@ -86,5 +85,5 @@ * */ -static inline uint8_t pio_read_8(ioport8_t *port) +NO_TRACE static inline uint8_t pio_read_8(ioport8_t *port) { uint8_t val; @@ -107,5 +106,5 @@ * */ -static inline uint16_t pio_read_16(ioport16_t *port) +NO_TRACE static inline uint16_t pio_read_16(ioport16_t *port) { uint16_t val; @@ -128,5 +127,5 @@ * */ -static inline uint32_t pio_read_32(ioport32_t *port) +NO_TRACE static inline uint32_t pio_read_32(ioport32_t *port) { uint32_t val; @@ -149,9 +148,10 @@ * */ -static inline void pio_write_8(ioport8_t *port, uint8_t val) +NO_TRACE static inline void pio_write_8(ioport8_t *port, uint8_t val) { asm volatile ( "outb %b[val], %w[port]\n" - :: [val] "a" (val), [port] "d" (port) + :: [val] "a" (val), + [port] "d" (port) ); } @@ -165,9 +165,10 @@ * */ -static inline void pio_write_16(ioport16_t *port, uint16_t val) +NO_TRACE static inline void pio_write_16(ioport16_t *port, uint16_t val) { asm volatile ( "outw %w[val], %w[port]\n" - :: [val] "a" (val), [port] "d" (port) + :: [val] "a" (val), + [port] "d" (port) ); } @@ -181,16 +182,19 @@ * */ -static inline void pio_write_32(ioport32_t *port, uint32_t val) +NO_TRACE static inline void pio_write_32(ioport32_t *port, uint32_t val) { asm volatile ( "outl %[val], %w[port]\n" - :: [val] "a" (val), [port] "d" (port) + :: [val] "a" (val), + [port] "d" (port) ); } /** Swap Hidden part of GS register with visible one */ -static inline void swapgs(void) -{ - asm volatile("swapgs"); +NO_TRACE static inline void swapgs(void) +{ + asm volatile ( + "swapgs" + ); } @@ -203,5 +207,5 @@ * */ -static inline ipl_t interrupts_enable(void) { +NO_TRACE static inline ipl_t interrupts_enable(void) { ipl_t v; @@ -224,5 +228,5 @@ * */ -static inline ipl_t interrupts_disable(void) { +NO_TRACE static inline ipl_t interrupts_disable(void) { ipl_t v; @@ -244,5 +248,5 @@ * */ -static inline void interrupts_restore(ipl_t ipl) { +NO_TRACE static inline void interrupts_restore(ipl_t ipl) { asm volatile ( "pushq %[ipl]\n" @@ -259,5 +263,5 @@ * */ -static inline ipl_t interrupts_read(void) { +NO_TRACE static inline ipl_t interrupts_read(void) { ipl_t v; @@ -276,5 +280,5 @@ * */ -static inline bool interrupts_disabled(void) +NO_TRACE static inline bool interrupts_disabled(void) { ipl_t v; @@ -289,7 +293,6 @@ } - /** Write to MSR */ -static inline void write_msr(uint32_t msr, uint64_t value) +NO_TRACE static inline void write_msr(uint32_t msr, uint64_t value) { asm volatile ( @@ -301,5 +304,5 @@ } -static inline unative_t read_msr(uint32_t msr) +NO_TRACE static inline unative_t read_msr(uint32_t msr) { uint32_t ax, dx; @@ -314,5 +317,4 @@ } - /** Enable local APIC * @@ -320,5 +322,5 @@ * */ -static inline void enable_l_apic_in_msr() +NO_TRACE static inline void enable_l_apic_in_msr() { asm volatile ( @@ -328,5 +330,5 @@ "orl $(0xfee00000),%%eax\n" "wrmsr\n" - ::: "%eax","%ecx","%edx" + ::: "%eax", "%ecx", "%edx" ); } @@ -337,5 +339,5 @@ * */ -static inline void invlpg(uintptr_t addr) +NO_TRACE static inline void invlpg(uintptr_t addr) { asm volatile ( @@ -350,5 +352,5 @@ * */ -static inline void gdtr_load(ptr_16_64_t *gdtr_reg) +NO_TRACE static inline void gdtr_load(ptr_16_64_t *gdtr_reg) { asm volatile ( @@ -363,5 +365,5 @@ * */ -static inline void gdtr_store(ptr_16_64_t *gdtr_reg) +NO_TRACE static inline void gdtr_store(ptr_16_64_t *gdtr_reg) { asm volatile ( @@ -376,5 +378,5 @@ * */ -static inline void idtr_load(ptr_16_64_t *idtr_reg) +NO_TRACE static inline void idtr_load(ptr_16_64_t *idtr_reg) { asm volatile ( @@ -388,5 +390,5 @@ * */ -static inline void tr_load(uint16_t sel) +NO_TRACE static inline void tr_load(uint16_t sel) { asm volatile ( @@ -396,5 +398,5 @@ } -#define GEN_READ_REG(reg) static inline unative_t read_ ##reg (void) \ +#define GEN_READ_REG(reg) NO_TRACE static inline unative_t read_ ##reg (void) \ { \ unative_t res; \ @@ -406,5 +408,5 @@ } -#define GEN_WRITE_REG(reg) static inline void write_ ##reg (unative_t regn) \ +#define GEN_WRITE_REG(reg) NO_TRACE static inline void write_ ##reg (unative_t regn) \ { \ asm volatile ( \ @@ -436,4 +438,7 @@ extern void interrupt_handlers(void); +extern void asm_delay_loop(uint32_t); +extern void asm_fake_loop(uint32_t); + #endif Index: kernel/arch/amd64/include/atomic.h =================================================================== --- kernel/arch/amd64/include/atomic.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/amd64/include/atomic.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -39,6 +39,7 @@ #include #include +#include -static inline void atomic_inc(atomic_t *val) +NO_TRACE static inline void atomic_inc(atomic_t *val) { #ifdef CONFIG_SMP @@ -55,5 +56,5 @@ } -static inline void atomic_dec(atomic_t *val) +NO_TRACE static inline void atomic_dec(atomic_t *val) { #ifdef CONFIG_SMP @@ -70,5 +71,5 @@ } -static inline atomic_count_t atomic_postinc(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) { atomic_count_t r = 1; @@ -83,5 +84,5 @@ } -static inline atomic_count_t atomic_postdec(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) { atomic_count_t r = -1; @@ -99,5 +100,5 @@ #define atomic_predec(val) (atomic_postdec(val) - 1) -static inline atomic_count_t test_and_set(atomic_t *val) +NO_TRACE static inline atomic_count_t test_and_set(atomic_t *val) { atomic_count_t v = 1; @@ -113,5 +114,5 @@ /** amd64 specific fast spinlock */ -static inline void atomic_lock_arch(atomic_t *val) +NO_TRACE static inline void atomic_lock_arch(atomic_t *val) { atomic_count_t tmp; @@ -120,13 +121,13 @@ asm volatile ( "0:\n" - "pause\n" - "mov %[count], %[tmp]\n" - "testq %[tmp], %[tmp]\n" - "jnz 0b\n" /* lightweight looping on locked spinlock */ + " pause\n" + " mov %[count], %[tmp]\n" + " testq %[tmp], %[tmp]\n" + " jnz 0b\n" /* lightweight looping on locked spinlock */ - "incq %[tmp]\n" /* now use the atomic operation */ - "xchgq %[count], %[tmp]\n" - "testq %[tmp], %[tmp]\n" - "jnz 0b\n" + " incq %[tmp]\n" /* now use the atomic operation */ + " xchgq %[count], %[tmp]\n" + " testq %[tmp], %[tmp]\n" + " jnz 0b\n" : [count] "+m" (val->count), [tmp] "=&r" (tmp) Index: kernel/arch/amd64/include/cycle.h =================================================================== --- kernel/arch/amd64/include/cycle.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/amd64/include/cycle.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -36,5 +36,7 @@ #define KERN_amd64_CYCLE_H_ -static inline uint64_t get_cycle(void) +#include + +NO_TRACE static inline uint64_t get_cycle(void) { uint32_t lower; Index: kernel/arch/amd64/include/interrupt.h =================================================================== --- kernel/arch/amd64/include/interrupt.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/amd64/include/interrupt.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -38,24 +38,25 @@ #include #include +#include -#define IVT_ITEMS IDT_ITEMS -#define IVT_FIRST 0 +#define IVT_ITEMS IDT_ITEMS +#define IVT_FIRST 0 -#define EXC_COUNT 32 -#define IRQ_COUNT 16 +#define EXC_COUNT 32 +#define IRQ_COUNT 16 -#define IVT_EXCBASE 0 -#define IVT_IRQBASE (IVT_EXCBASE + EXC_COUNT) -#define IVT_FREEBASE (IVT_IRQBASE + IRQ_COUNT) +#define IVT_EXCBASE 0 +#define IVT_IRQBASE (IVT_EXCBASE + EXC_COUNT) +#define IVT_FREEBASE (IVT_IRQBASE + IRQ_COUNT) -#define IRQ_CLK 0 -#define IRQ_KBD 1 -#define IRQ_PIC1 2 -#define IRQ_PIC_SPUR 7 -#define IRQ_MOUSE 12 -#define IRQ_DP8390 9 +#define IRQ_CLK 0 +#define IRQ_KBD 1 +#define IRQ_PIC1 2 +#define IRQ_PIC_SPUR 7 +#define IRQ_MOUSE 12 +#define IRQ_DP8390 9 -/* this one must have four least significant bits set to ones */ -#define VECTOR_APIC_SPUR (IVT_ITEMS - 1) +/* This one must have four least significant bits set to ones */ +#define VECTOR_APIC_SPUR (IVT_ITEMS - 1) #if (((VECTOR_APIC_SPUR + 1) % 16) || VECTOR_APIC_SPUR >= IVT_ITEMS) @@ -63,10 +64,10 @@ #endif -#define VECTOR_DEBUG 1 -#define VECTOR_CLK (IVT_IRQBASE + IRQ_CLK) -#define VECTOR_PIC_SPUR (IVT_IRQBASE + IRQ_PIC_SPUR) -#define VECTOR_SYSCALL IVT_FREEBASE -#define VECTOR_TLB_SHOOTDOWN_IPI (IVT_FREEBASE + 1) -#define VECTOR_DEBUG_IPI (IVT_FREEBASE + 2) +#define VECTOR_DEBUG 1 +#define VECTOR_CLK (IVT_IRQBASE + IRQ_CLK) +#define VECTOR_PIC_SPUR (IVT_IRQBASE + IRQ_PIC_SPUR) +#define VECTOR_SYSCALL IVT_FREEBASE +#define VECTOR_TLB_SHOOTDOWN_IPI (IVT_FREEBASE + 1) +#define VECTOR_DEBUG_IPI (IVT_FREEBASE + 2) /** This is passed to interrupt handlers */ @@ -86,33 +87,36 @@ uint64_t cs; uint64_t rflags; - uint64_t stack[]; /* Additional data on stack */ + uint64_t stack[]; /* Additional data on stack */ } istate_t; /** Return true if exception happened while in userspace */ -static inline int istate_from_uspace(istate_t *istate) +NO_TRACE static inline int istate_from_uspace(istate_t *istate) { return !(istate->rip & 0x8000000000000000); } -static inline void istate_set_retaddr(istate_t *istate, uintptr_t retaddr) +NO_TRACE static inline void istate_set_retaddr(istate_t *istate, + uintptr_t retaddr) { istate->rip = retaddr; } -static inline unative_t istate_get_pc(istate_t *istate) + +NO_TRACE static inline unative_t istate_get_pc(istate_t *istate) { return istate->rip; } -static inline unative_t istate_get_fp(istate_t *istate) + +NO_TRACE static inline unative_t istate_get_fp(istate_t *istate) { return istate->rbp; } -extern void (* disable_irqs_function)(uint16_t irqmask); -extern void (* enable_irqs_function)(uint16_t irqmask); +extern void (* disable_irqs_function)(uint16_t); +extern void (* enable_irqs_function)(uint16_t); extern void (* eoi_function)(void); extern void interrupt_init(void); -extern void trap_virtual_enable_irqs(uint16_t irqmask); -extern void trap_virtual_disable_irqs(uint16_t irqmask); +extern void trap_virtual_enable_irqs(uint16_t); +extern void trap_virtual_disable_irqs(uint16_t); #endif Index: kernel/arch/amd64/include/mm/frame.h =================================================================== --- kernel/arch/amd64/include/mm/frame.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/amd64/include/mm/frame.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -36,16 +36,18 @@ #define KERN_amd64_FRAME_H_ -#ifndef __ASM__ -#include -#endif /* __ASM__ */ - #define FRAME_WIDTH 12 /* 4K */ #define FRAME_SIZE (1 << FRAME_WIDTH) +#ifdef KERNEL #ifndef __ASM__ + +#include + extern uintptr_t last_frame; extern void frame_arch_init(void); extern void physmem_print(void); + #endif /* __ASM__ */ +#endif /* KERNEL */ #endif Index: kernel/arch/amd64/include/mm/page.h =================================================================== --- kernel/arch/amd64/include/mm/page.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/amd64/include/mm/page.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -46,4 +46,5 @@ #include +#include #define PAGE_WIDTH FRAME_WIDTH @@ -187,5 +188,5 @@ } __attribute__ ((packed)) pte_t; -static inline unsigned int get_pt_flags(pte_t *pt, size_t i) +NO_TRACE static inline unsigned int get_pt_flags(pte_t *pt, size_t i) { pte_t *p = &pt[i]; @@ -200,5 +201,5 @@ } -static inline void set_pt_addr(pte_t *pt, size_t i, uintptr_t a) +NO_TRACE static inline void set_pt_addr(pte_t *pt, size_t i, uintptr_t a) { pte_t *p = &pt[i]; @@ -208,5 +209,5 @@ } -static inline void set_pt_flags(pte_t *pt, size_t i, int flags) +NO_TRACE static inline void set_pt_flags(pte_t *pt, size_t i, int flags) { pte_t *p = &pt[i]; Index: kernel/arch/arm32/include/asm.h =================================================================== --- kernel/arch/arm32/include/asm.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/arm32/include/asm.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -41,36 +41,37 @@ #include #include +#include /** No such instruction on ARM to sleep CPU. */ -static inline void cpu_sleep(void) +NO_TRACE static inline void cpu_sleep(void) { } -static inline void pio_write_8(ioport8_t *port, uint8_t v) +NO_TRACE static inline void pio_write_8(ioport8_t *port, uint8_t v) { *port = v; } -static inline void pio_write_16(ioport16_t *port, uint16_t v) +NO_TRACE static inline void pio_write_16(ioport16_t *port, uint16_t v) { *port = v; } -static inline void pio_write_32(ioport32_t *port, uint32_t v) +NO_TRACE static inline void pio_write_32(ioport32_t *port, uint32_t v) { *port = v; } -static inline uint8_t pio_read_8(ioport8_t *port) +NO_TRACE static inline uint8_t pio_read_8(ioport8_t *port) { return *port; } -static inline uint16_t pio_read_16(ioport16_t *port) +NO_TRACE static inline uint16_t pio_read_16(ioport16_t *port) { return *port; } -static inline uint32_t pio_read_32(ioport32_t *port) +NO_TRACE static inline uint32_t pio_read_32(ioport32_t *port) { return *port; @@ -84,7 +85,8 @@ * */ -static inline uintptr_t get_stack_base(void) +NO_TRACE static inline uintptr_t get_stack_base(void) { uintptr_t v; + asm volatile ( "and %[v], sp, %[size]\n" @@ -92,4 +94,5 @@ : [size] "r" (~(STACK_SIZE - 1)) ); + return v; } Index: kernel/arch/arm32/include/atomic.h =================================================================== --- kernel/arch/arm32/include/atomic.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/arm32/include/atomic.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -38,4 +38,5 @@ #include +#include /** Atomic addition. @@ -47,5 +48,6 @@ * */ -static inline atomic_count_t atomic_add(atomic_t *val, atomic_count_t i) +NO_TRACE static inline atomic_count_t atomic_add(atomic_t *val, + atomic_count_t i) { /* @@ -66,5 +68,5 @@ * */ -static inline void atomic_inc(atomic_t *val) +NO_TRACE static inline void atomic_inc(atomic_t *val) { atomic_add(val, 1); @@ -76,5 +78,5 @@ * */ -static inline void atomic_dec(atomic_t *val) { +NO_TRACE static inline void atomic_dec(atomic_t *val) { atomic_add(val, -1); } @@ -86,5 +88,5 @@ * */ -static inline atomic_count_t atomic_preinc(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_preinc(atomic_t *val) { return atomic_add(val, 1); @@ -97,5 +99,5 @@ * */ -static inline atomic_count_t atomic_predec(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_predec(atomic_t *val) { return atomic_add(val, -1); @@ -108,5 +110,5 @@ * */ -static inline atomic_count_t atomic_postinc(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) { return atomic_add(val, 1) - 1; @@ -119,5 +121,5 @@ * */ -static inline atomic_count_t atomic_postdec(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) { return atomic_add(val, -1) + 1; Index: kernel/arch/arm32/include/cycle.h =================================================================== --- kernel/arch/arm32/include/cycle.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/arm32/include/cycle.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -37,11 +37,14 @@ #define KERN_arm32_CYCLE_H_ -/** Returns count of CPU cycles. +#include + +/** Return count of CPU cycles. * - * No such instruction on ARM to get count of cycles. + * No such instruction on ARM to get count of cycles. * - * @return Count of CPU cycles. + * @return Count of CPU cycles. + * */ -static inline uint64_t get_cycle(void) +NO_TRACE static inline uint64_t get_cycle(void) { return 0; Index: kernel/arch/arm32/include/exception.h =================================================================== --- kernel/arch/arm32/include/exception.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/arm32/include/exception.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -40,4 +40,5 @@ #include #include +#include /** If defined, forces using of high exception vectors. */ @@ -45,17 +46,17 @@ #ifdef HIGH_EXCEPTION_VECTORS - #define EXC_BASE_ADDRESS 0xffff0000 + #define EXC_BASE_ADDRESS 0xffff0000 #else - #define EXC_BASE_ADDRESS 0x0 + #define EXC_BASE_ADDRESS 0x0 #endif /* Exception Vectors */ -#define EXC_RESET_VEC (EXC_BASE_ADDRESS + 0x0) -#define EXC_UNDEF_INSTR_VEC (EXC_BASE_ADDRESS + 0x4) -#define EXC_SWI_VEC (EXC_BASE_ADDRESS + 0x8) -#define EXC_PREFETCH_ABORT_VEC (EXC_BASE_ADDRESS + 0xc) -#define EXC_DATA_ABORT_VEC (EXC_BASE_ADDRESS + 0x10) -#define EXC_IRQ_VEC (EXC_BASE_ADDRESS + 0x18) -#define EXC_FIQ_VEC (EXC_BASE_ADDRESS + 0x1c) +#define EXC_RESET_VEC (EXC_BASE_ADDRESS + 0x0) +#define EXC_UNDEF_INSTR_VEC (EXC_BASE_ADDRESS + 0x4) +#define EXC_SWI_VEC (EXC_BASE_ADDRESS + 0x8) +#define EXC_PREFETCH_ABORT_VEC (EXC_BASE_ADDRESS + 0xc) +#define EXC_DATA_ABORT_VEC (EXC_BASE_ADDRESS + 0x10) +#define EXC_IRQ_VEC (EXC_BASE_ADDRESS + 0x18) +#define EXC_FIQ_VEC (EXC_BASE_ADDRESS + 0x1c) /* Exception numbers */ @@ -68,12 +69,11 @@ #define EXC_FIQ 6 - /** Kernel stack pointer. * * It is set when thread switches to user mode, * and then used for exception handling. + * */ extern uintptr_t supervisor_sp; - /** Temporary exception stack pointer. @@ -81,7 +81,7 @@ * Temporary stack is used in exceptions handling routines * before switching to thread's kernel stack. + * */ extern uintptr_t exc_stack; - /** Struct representing CPU state saved when an exception occurs. */ @@ -90,5 +90,5 @@ uint32_t sp; uint32_t lr; - + uint32_t r0; uint32_t r1; @@ -104,38 +104,36 @@ uint32_t fp; uint32_t r12; - + uint32_t pc; } istate_t; - -/** Sets Program Counter member of given istate structure. +/** Set Program Counter member of given istate structure. * - * @param istate istate structure + * @param istate istate structure * @param retaddr new value of istate's PC member + * */ -static inline void istate_set_retaddr(istate_t *istate, uintptr_t retaddr) +NO_TRACE static inline void istate_set_retaddr(istate_t *istate, + uintptr_t retaddr) { - istate->pc = retaddr; + istate->pc = retaddr; } - -/** Returns true if exception happened while in userspace. */ -static inline int istate_from_uspace(istate_t *istate) +/** Return true if exception happened while in userspace. */ +NO_TRACE static inline int istate_from_uspace(istate_t *istate) { - return (istate->spsr & STATUS_REG_MODE_MASK) == USER_MODE; + return (istate->spsr & STATUS_REG_MODE_MASK) == USER_MODE; } - -/** Returns Program Counter member of given istate structure. */ -static inline unative_t istate_get_pc(istate_t *istate) +/** Return Program Counter member of given istate structure. */ +NO_TRACE static inline unative_t istate_get_pc(istate_t *istate) { - return istate->pc; + return istate->pc; } -static inline unative_t istate_get_fp(istate_t *istate) +NO_TRACE static inline unative_t istate_get_fp(istate_t *istate) { return istate->fp; } - extern void install_exception_handlers(void); @@ -149,5 +147,4 @@ extern void swi_exception_entry(void); - #endif Index: kernel/arch/arm32/include/faddr.h =================================================================== --- kernel/arch/arm32/include/faddr.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/arm32/include/faddr.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup arm32 +/** @addtogroup arm32 * @{ */ @@ -42,6 +42,7 @@ * * @param fptr Function pointer. + * */ -#define FADDR(fptr) ((uintptr_t) (fptr)) +#define FADDR(fptr) ((uintptr_t) (fptr)) #endif Index: kernel/arch/arm32/include/interrupt.h =================================================================== --- kernel/arch/arm32/include/interrupt.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/arm32/include/interrupt.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -41,9 +41,8 @@ /** Initial size of exception dispatch table. */ -#define IVT_ITEMS 6 +#define IVT_ITEMS 6 /** Index of the first item in exception dispatch table. */ -#define IVT_FIRST 0 - +#define IVT_FIRST 0 extern void interrupt_init(void); @@ -54,5 +53,4 @@ extern bool interrupts_disabled(void); - #endif Index: kernel/arch/arm32/include/mm/page.h =================================================================== --- kernel/arch/arm32/include/mm/page.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/arm32/include/mm/page.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup arm32mm +/** @addtogroup arm32mm * @{ */ @@ -40,4 +40,5 @@ #include #include +#include #define PAGE_WIDTH FRAME_WIDTH @@ -192,7 +193,8 @@ /** Sets the address of level 0 page table. * - * @param pt Pointer to the page table to set. - */ -static inline void set_ptl0_addr(pte_t *pt) + * @param pt Pointer to the page table to set. + * + */ +NO_TRACE static inline void set_ptl0_addr(pte_t *pt) { asm volatile ( @@ -205,12 +207,13 @@ /** Returns level 0 page table entry flags. * - * @param pt Level 0 page table. - * @param i Index of the entry to return. - */ -static inline int get_pt_level0_flags(pte_t *pt, size_t i) + * @param pt Level 0 page table. + * @param i Index of the entry to return. + * + */ +NO_TRACE static inline int get_pt_level0_flags(pte_t *pt, size_t i) { pte_level0_t *p = &pt[i].l0; int np = (p->descriptor_type == PTE_DESCRIPTOR_NOT_PRESENT); - + return (np << PAGE_PRESENT_SHIFT) | (1 << PAGE_USER_SHIFT) | (1 << PAGE_READ_SHIFT) | (1 << PAGE_WRITE_SHIFT) | @@ -220,14 +223,15 @@ /** Returns level 1 page table entry flags. * - * @param pt Level 1 page table. - * @param i Index of the entry to return. - */ -static inline int get_pt_level1_flags(pte_t *pt, size_t i) + * @param pt Level 1 page table. + * @param i Index of the entry to return. + * + */ +NO_TRACE static inline int get_pt_level1_flags(pte_t *pt, size_t i) { pte_level1_t *p = &pt[i].l1; - + int dt = p->descriptor_type; int ap = p->access_permission_0; - + return ((dt == PTE_DESCRIPTOR_NOT_PRESENT) << PAGE_PRESENT_SHIFT) | ((ap == PTE_AP_USER_RO_KERNEL_RW) << PAGE_READ_SHIFT) | @@ -241,15 +245,15 @@ } - /** Sets flags of level 0 page table entry. * - * @param pt level 0 page table - * @param i index of the entry to be changed - * @param flags new flags - */ -static inline void set_pt_level0_flags(pte_t *pt, size_t i, int flags) + * @param pt level 0 page table + * @param i index of the entry to be changed + * @param flags new flags + * + */ +NO_TRACE static inline void set_pt_level0_flags(pte_t *pt, size_t i, int flags) { pte_level0_t *p = &pt[i].l0; - + if (flags & PAGE_NOT_PRESENT) { p->descriptor_type = PTE_DESCRIPTOR_NOT_PRESENT; @@ -262,5 +266,5 @@ p->descriptor_type = PTE_DESCRIPTOR_COARSE_TABLE; p->should_be_zero = 0; - } + } } @@ -268,13 +272,14 @@ /** Sets flags of level 1 page table entry. * - * We use same access rights for the whole page. When page is not preset we - * store 1 in acess_rigts_3 so that at least one bit is 1 (to mark correct - * page entry, see #PAGE_VALID_ARCH). - * - * @param pt Level 1 page table. - * @param i Index of the entry to be changed. - * @param flags New flags. - */ -static inline void set_pt_level1_flags(pte_t *pt, size_t i, int flags) + * We use same access rights for the whole page. When page + * is not preset we store 1 in acess_rigts_3 so that at least + * one bit is 1 (to mark correct page entry, see #PAGE_VALID_ARCH). + * + * @param pt Level 1 page table. + * @param i Index of the entry to be changed. + * @param flags New flags. + * + */ +NO_TRACE static inline void set_pt_level1_flags(pte_t *pt, size_t i, int flags) { pte_level1_t *p = &pt[i].l1; @@ -287,12 +292,12 @@ p->access_permission_3 = p->access_permission_0; } - + p->cacheable = p->bufferable = (flags & PAGE_CACHEABLE) != 0; - + /* default access permission */ p->access_permission_0 = p->access_permission_1 = p->access_permission_2 = p->access_permission_3 = PTE_AP_USER_NO_KERNEL_RW; - + if (flags & PAGE_USER) { if (flags & PAGE_READ) { Index: kernel/arch/arm32/include/mm/tlb.h =================================================================== --- kernel/arch/arm32/include/mm/tlb.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/arm32/include/mm/tlb.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup arm32mm +/** @addtogroup arm32mm * @{ */ Index: kernel/arch/ia32/include/asm.h =================================================================== --- kernel/arch/ia32/include/asm.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ia32/include/asm.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -41,18 +41,8 @@ #include #include +#include extern uint32_t interrupt_handler_size; -extern void paging_on(void); - -extern void interrupt_handlers(void); - -extern void enable_l_apic_in_msr(void); - - -extern void asm_delay_loop(uint32_t t); -extern void asm_fake_loop(uint32_t t); - - /** Halt CPU * @@ -60,5 +50,5 @@ * */ -static inline __attribute__((noreturn)) void cpu_halt(void) +NO_TRACE static inline __attribute__((noreturn)) void cpu_halt(void) { while (true) { @@ -69,10 +59,12 @@ } -static inline void cpu_sleep(void) -{ - asm volatile ("hlt\n"); -} - -#define GEN_READ_REG(reg) static inline unative_t read_ ##reg (void) \ +NO_TRACE static inline void cpu_sleep(void) +{ + asm volatile ( + "hlt\n" + ); +} + +#define GEN_READ_REG(reg) NO_TRACE static inline unative_t read_ ##reg (void) \ { \ unative_t res; \ @@ -84,5 +76,5 @@ } -#define GEN_WRITE_REG(reg) static inline void write_ ##reg (unative_t regn) \ +#define GEN_WRITE_REG(reg) NO_TRACE static inline void write_ ##reg (unative_t regn) \ { \ asm volatile ( \ @@ -119,9 +111,10 @@ * */ -static inline void pio_write_8(ioport8_t *port, uint8_t val) +NO_TRACE static inline void pio_write_8(ioport8_t *port, uint8_t val) { asm volatile ( "outb %b[val], %w[port]\n" - :: [val] "a" (val), [port] "d" (port) + :: [val] "a" (val), + [port] "d" (port) ); } @@ -135,9 +128,10 @@ * */ -static inline void pio_write_16(ioport16_t *port, uint16_t val) +NO_TRACE static inline void pio_write_16(ioport16_t *port, uint16_t val) { asm volatile ( "outw %w[val], %w[port]\n" - :: [val] "a" (val), [port] "d" (port) + :: [val] "a" (val), + [port] "d" (port) ); } @@ -151,9 +145,10 @@ * */ -static inline void pio_write_32(ioport32_t *port, uint32_t val) +NO_TRACE static inline void pio_write_32(ioport32_t *port, uint32_t val) { asm volatile ( "outl %[val], %w[port]\n" - :: [val] "a" (val), [port] "d" (port) + :: [val] "a" (val), + [port] "d" (port) ); } @@ -167,5 +162,5 @@ * */ -static inline uint8_t pio_read_8(ioport8_t *port) +NO_TRACE static inline uint8_t pio_read_8(ioport8_t *port) { uint8_t val; @@ -188,5 +183,5 @@ * */ -static inline uint16_t pio_read_16(ioport16_t *port) +NO_TRACE static inline uint16_t pio_read_16(ioport16_t *port) { uint16_t val; @@ -209,5 +204,5 @@ * */ -static inline uint32_t pio_read_32(ioport32_t *port) +NO_TRACE static inline uint32_t pio_read_32(ioport32_t *port) { uint32_t val; @@ -230,5 +225,5 @@ * */ -static inline ipl_t interrupts_enable(void) +NO_TRACE static inline ipl_t interrupts_enable(void) { ipl_t v; @@ -252,5 +247,5 @@ * */ -static inline ipl_t interrupts_disable(void) +NO_TRACE static inline ipl_t interrupts_disable(void) { ipl_t v; @@ -273,5 +268,5 @@ * */ -static inline void interrupts_restore(ipl_t ipl) +NO_TRACE static inline void interrupts_restore(ipl_t ipl) { asm volatile ( @@ -287,5 +282,5 @@ * */ -static inline ipl_t interrupts_read(void) +NO_TRACE static inline ipl_t interrupts_read(void) { ipl_t v; @@ -305,5 +300,5 @@ * */ -static inline bool interrupts_disabled(void) +NO_TRACE static inline bool interrupts_disabled(void) { ipl_t v; @@ -319,14 +314,15 @@ /** Write to MSR */ -static inline void write_msr(uint32_t msr, uint64_t value) +NO_TRACE static inline void write_msr(uint32_t msr, uint64_t value) { asm volatile ( "wrmsr" - :: "c" (msr), "a" ((uint32_t) (value)), + :: "c" (msr), + "a" ((uint32_t) (value)), "d" ((uint32_t) (value >> 32)) ); } -static inline uint64_t read_msr(uint32_t msr) +NO_TRACE static inline uint64_t read_msr(uint32_t msr) { uint32_t ax, dx; @@ -334,5 +330,6 @@ asm volatile ( "rdmsr" - : "=a" (ax), "=d" (dx) + : "=a" (ax), + "=d" (dx) : "c" (msr) ); @@ -349,5 +346,5 @@ * */ -static inline uintptr_t get_stack_base(void) +NO_TRACE static inline uintptr_t get_stack_base(void) { uintptr_t v; @@ -367,5 +364,5 @@ * */ -static inline void invlpg(uintptr_t addr) +NO_TRACE static inline void invlpg(uintptr_t addr) { asm volatile ( @@ -380,5 +377,5 @@ * */ -static inline void gdtr_load(ptr_16_32_t *gdtr_reg) +NO_TRACE static inline void gdtr_load(ptr_16_32_t *gdtr_reg) { asm volatile ( @@ -393,5 +390,5 @@ * */ -static inline void gdtr_store(ptr_16_32_t *gdtr_reg) +NO_TRACE static inline void gdtr_store(ptr_16_32_t *gdtr_reg) { asm volatile ( @@ -406,5 +403,5 @@ * */ -static inline void idtr_load(ptr_16_32_t *idtr_reg) +NO_TRACE static inline void idtr_load(ptr_16_32_t *idtr_reg) { asm volatile ( @@ -419,5 +416,5 @@ * */ -static inline void tr_load(uint16_t sel) +NO_TRACE static inline void tr_load(uint16_t sel) { asm volatile ( @@ -427,4 +424,11 @@ } +extern void paging_on(void); +extern void interrupt_handlers(void); +extern void enable_l_apic_in_msr(void); + +extern void asm_delay_loop(uint32_t); +extern void asm_fake_loop(uint32_t); + #endif Index: kernel/arch/ia32/include/atomic.h =================================================================== --- kernel/arch/ia32/include/atomic.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ia32/include/atomic.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -39,6 +39,7 @@ #include #include +#include -static inline void atomic_inc(atomic_t *val) +NO_TRACE static inline void atomic_inc(atomic_t *val) { #ifdef CONFIG_SMP @@ -55,5 +56,5 @@ } -static inline void atomic_dec(atomic_t *val) +NO_TRACE static inline void atomic_dec(atomic_t *val) { #ifdef CONFIG_SMP @@ -70,5 +71,5 @@ } -static inline atomic_count_t atomic_postinc(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) { atomic_count_t r = 1; @@ -83,5 +84,5 @@ } -static inline atomic_count_t atomic_postdec(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) { atomic_count_t r = -1; @@ -99,5 +100,5 @@ #define atomic_predec(val) (atomic_postdec(val) - 1) -static inline atomic_count_t test_and_set(atomic_t *val) +NO_TRACE static inline atomic_count_t test_and_set(atomic_t *val) { atomic_count_t v = 1; @@ -113,5 +114,5 @@ /** ia32 specific fast spinlock */ -static inline void atomic_lock_arch(atomic_t *val) +NO_TRACE static inline void atomic_lock_arch(atomic_t *val) { atomic_count_t tmp; Index: kernel/arch/ia32/include/barrier.h =================================================================== --- kernel/arch/ia32/include/barrier.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ia32/include/barrier.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -36,4 +36,6 @@ #define KERN_ia32_BARRIER_H_ +#include + /* * NOTE: @@ -50,5 +52,5 @@ #define CS_LEAVE_BARRIER() asm volatile ("" ::: "memory") -static inline void cpuid_serialization(void) +NO_TRACE static inline void cpuid_serialization(void) { asm volatile ( Index: kernel/arch/ia32/include/cycle.h =================================================================== --- kernel/arch/ia32/include/cycle.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ia32/include/cycle.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup ia32 +/** @addtogroup ia32 * @{ */ @@ -36,5 +36,7 @@ #define KERN_ia32_CYCLE_H_ -static inline uint64_t get_cycle(void) +#include + +NO_TRACE static inline uint64_t get_cycle(void) { uint64_t v; Index: kernel/arch/ia32/include/interrupt.h =================================================================== --- kernel/arch/ia32/include/interrupt.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ia32/include/interrupt.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -38,24 +38,25 @@ #include #include +#include -#define IVT_ITEMS IDT_ITEMS -#define IVT_FIRST 0 +#define IVT_ITEMS IDT_ITEMS +#define IVT_FIRST 0 -#define EXC_COUNT 32 -#define IRQ_COUNT 16 +#define EXC_COUNT 32 +#define IRQ_COUNT 16 -#define IVT_EXCBASE 0 -#define IVT_IRQBASE (IVT_EXCBASE + EXC_COUNT) -#define IVT_FREEBASE (IVT_IRQBASE + IRQ_COUNT) +#define IVT_EXCBASE 0 +#define IVT_IRQBASE (IVT_EXCBASE + EXC_COUNT) +#define IVT_FREEBASE (IVT_IRQBASE + IRQ_COUNT) -#define IRQ_CLK 0 -#define IRQ_KBD 1 -#define IRQ_PIC1 2 -#define IRQ_PIC_SPUR 7 -#define IRQ_MOUSE 12 -#define IRQ_DP8390 9 +#define IRQ_CLK 0 +#define IRQ_KBD 1 +#define IRQ_PIC1 2 +#define IRQ_PIC_SPUR 7 +#define IRQ_MOUSE 12 +#define IRQ_DP8390 9 -/* this one must have four least significant bits set to ones */ -#define VECTOR_APIC_SPUR (IVT_ITEMS - 1) +/* This one must have four least significant bits set to ones */ +#define VECTOR_APIC_SPUR (IVT_ITEMS - 1) #if (((VECTOR_APIC_SPUR + 1) % 16) || VECTOR_APIC_SPUR >= IVT_ITEMS) @@ -63,10 +64,10 @@ #endif -#define VECTOR_DEBUG 1 -#define VECTOR_CLK (IVT_IRQBASE + IRQ_CLK) -#define VECTOR_PIC_SPUR (IVT_IRQBASE + IRQ_PIC_SPUR) -#define VECTOR_SYSCALL IVT_FREEBASE -#define VECTOR_TLB_SHOOTDOWN_IPI (IVT_FREEBASE + 1) -#define VECTOR_DEBUG_IPI (IVT_FREEBASE + 2) +#define VECTOR_DEBUG 1 +#define VECTOR_CLK (IVT_IRQBASE + IRQ_CLK) +#define VECTOR_PIC_SPUR (IVT_IRQBASE + IRQ_PIC_SPUR) +#define VECTOR_SYSCALL IVT_FREEBASE +#define VECTOR_TLB_SHOOTDOWN_IPI (IVT_FREEBASE + 1) +#define VECTOR_DEBUG_IPI (IVT_FREEBASE + 2) typedef struct istate { @@ -79,48 +80,49 @@ uint32_t ebp; - uint32_t ebp_frame; /* imitation of frame pointer linkage */ - uint32_t eip_frame; /* imitation of return address linkage */ - + uint32_t ebp_frame; /* imitation of frame pointer linkage */ + uint32_t eip_frame; /* imitation of return address linkage */ + uint32_t gs; uint32_t fs; uint32_t es; uint32_t ds; - - uint32_t error_word; /* real or fake error word */ + + uint32_t error_word; /* real or fake error word */ uint32_t eip; uint32_t cs; uint32_t eflags; - uint32_t esp; /* only if istate_t is from uspace */ - uint32_t ss; /* only if istate_t is from uspace */ + uint32_t esp; /* only if istate_t is from uspace */ + uint32_t ss; /* only if istate_t is from uspace */ } istate_t; /** Return true if exception happened while in userspace */ -static inline int istate_from_uspace(istate_t *istate) +NO_TRACE static inline int istate_from_uspace(istate_t *istate) { return !(istate->eip & 0x80000000); } -static inline void istate_set_retaddr(istate_t *istate, uintptr_t retaddr) +NO_TRACE static inline void istate_set_retaddr(istate_t *istate, + uintptr_t retaddr) { istate->eip = retaddr; } -static inline unative_t istate_get_pc(istate_t *istate) +NO_TRACE static inline unative_t istate_get_pc(istate_t *istate) { return istate->eip; } -static inline unative_t istate_get_fp(istate_t *istate) +NO_TRACE static inline unative_t istate_get_fp(istate_t *istate) { return istate->ebp; } -extern void (* disable_irqs_function)(uint16_t irqmask); -extern void (* enable_irqs_function)(uint16_t irqmask); +extern void (* disable_irqs_function)(uint16_t); +extern void (* enable_irqs_function)(uint16_t); extern void (* eoi_function)(void); extern void interrupt_init(void); -extern void trap_virtual_enable_irqs(uint16_t irqmask); -extern void trap_virtual_disable_irqs(uint16_t irqmask); +extern void trap_virtual_enable_irqs(uint16_t); +extern void trap_virtual_disable_irqs(uint16_t); #endif Index: kernel/arch/ia32/include/mm/page.h =================================================================== --- kernel/arch/ia32/include/mm/page.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ia32/include/mm/page.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -37,4 +37,5 @@ #include +#include #define PAGE_WIDTH FRAME_WIDTH @@ -161,5 +162,5 @@ } __attribute__ ((packed)) pte_t; -static inline unsigned int get_pt_flags(pte_t *pt, size_t i) +NO_TRACE static inline unsigned int get_pt_flags(pte_t *pt, size_t i) { pte_t *p = &pt[i]; @@ -174,5 +175,5 @@ } -static inline void set_pt_flags(pte_t *pt, size_t i, int flags) +NO_TRACE static inline void set_pt_flags(pte_t *pt, size_t i, int flags) { pte_t *p = &pt[i]; Index: kernel/arch/ia64/include/asm.h =================================================================== --- kernel/arch/ia64/include/asm.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ia64/include/asm.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -40,8 +40,9 @@ #include #include +#include #define IA64_IOSPACE_ADDRESS 0xE001000000000000ULL -static inline void pio_write_8(ioport8_t *port, uint8_t v) +NO_TRACE static inline void pio_write_8(ioport8_t *port, uint8_t v) { uintptr_t prt = (uintptr_t) port; @@ -56,5 +57,5 @@ } -static inline void pio_write_16(ioport16_t *port, uint16_t v) +NO_TRACE static inline void pio_write_16(ioport16_t *port, uint16_t v) { uintptr_t prt = (uintptr_t) port; @@ -69,5 +70,5 @@ } -static inline void pio_write_32(ioport32_t *port, uint32_t v) +NO_TRACE static inline void pio_write_32(ioport32_t *port, uint32_t v) { uintptr_t prt = (uintptr_t) port; @@ -82,5 +83,5 @@ } -static inline uint8_t pio_read_8(ioport8_t *port) +NO_TRACE static inline uint8_t pio_read_8(ioport8_t *port) { uintptr_t prt = (uintptr_t) port; @@ -95,5 +96,5 @@ } -static inline uint16_t pio_read_16(ioport16_t *port) +NO_TRACE static inline uint16_t pio_read_16(ioport16_t *port) { uintptr_t prt = (uintptr_t) port; @@ -108,5 +109,5 @@ } -static inline uint32_t pio_read_32(ioport32_t *port) +NO_TRACE static inline uint32_t pio_read_32(ioport32_t *port) { uintptr_t prt = (uintptr_t) port; @@ -126,21 +127,26 @@ * The stack is assumed to be STACK_SIZE long. * The stack must start on page boundary. - */ -static inline uintptr_t get_stack_base(void) -{ - uint64_t v; - - /* I'm not sure why but this code bad inlines in scheduler, - so THE shifts about 16B and causes kernel panic - - asm volatile ( - "and %[value] = %[mask], r12" - : [value] "=r" (v) - : [mask] "r" (~(STACK_SIZE - 1)) - ); - return v; - - This code have the same meaning but inlines well. - */ + * + */ +NO_TRACE static inline uintptr_t get_stack_base(void) +{ + uint64_t v; + + /* + * I'm not sure why but this code inlines badly + * in scheduler, resulting in THE shifting about + * 16B and causing kernel panic. + * + * asm volatile ( + * "and %[value] = %[mask], r12" + * : [value] "=r" (v) + * : [mask] "r" (~(STACK_SIZE - 1)) + * ); + * return v; + * + * The following code has the same semantics but + * inlines correctly. + * + */ asm volatile ( @@ -155,6 +161,7 @@ * * @return PSR. - */ -static inline uint64_t psr_read(void) + * + */ +NO_TRACE static inline uint64_t psr_read(void) { uint64_t v; @@ -171,6 +178,7 @@ * * @return Return location of interruption vector table. - */ -static inline uint64_t iva_read(void) + * + */ +NO_TRACE static inline uint64_t iva_read(void) { uint64_t v; @@ -187,6 +195,7 @@ * * @param v New location of interruption vector table. - */ -static inline void iva_write(uint64_t v) + * + */ +NO_TRACE static inline void iva_write(uint64_t v) { asm volatile ( @@ -196,10 +205,11 @@ } - /** Read IVR (External Interrupt Vector Register). * - * @return Highest priority, pending, unmasked external interrupt vector. - */ -static inline uint64_t ivr_read(void) + * @return Highest priority, pending, unmasked external + * interrupt vector. + * + */ +NO_TRACE static inline uint64_t ivr_read(void) { uint64_t v; @@ -213,5 +223,5 @@ } -static inline uint64_t cr64_read(void) +NO_TRACE static inline uint64_t cr64_read(void) { uint64_t v; @@ -225,10 +235,10 @@ } - /** Write ITC (Interval Timer Counter) register. * * @param v New counter value. - */ -static inline void itc_write(uint64_t v) + * + */ +NO_TRACE static inline void itc_write(uint64_t v) { asm volatile ( @@ -241,6 +251,7 @@ * * @return Current counter value. - */ -static inline uint64_t itc_read(void) + * + */ +NO_TRACE static inline uint64_t itc_read(void) { uint64_t v; @@ -257,6 +268,7 @@ * * @param v New match value. - */ -static inline void itm_write(uint64_t v) + * + */ +NO_TRACE static inline void itm_write(uint64_t v) { asm volatile ( @@ -269,6 +281,7 @@ * * @return Match value. - */ -static inline uint64_t itm_read(void) + * + */ +NO_TRACE static inline uint64_t itm_read(void) { uint64_t v; @@ -285,6 +298,7 @@ * * @return Current vector and mask bit. - */ -static inline uint64_t itv_read(void) + * + */ +NO_TRACE static inline uint64_t itv_read(void) { uint64_t v; @@ -301,6 +315,7 @@ * * @param v New vector and mask bit. - */ -static inline void itv_write(uint64_t v) + * + */ +NO_TRACE static inline void itv_write(uint64_t v) { asm volatile ( @@ -313,6 +328,7 @@ * * @param v This value is ignored. - */ -static inline void eoi_write(uint64_t v) + * + */ +NO_TRACE static inline void eoi_write(uint64_t v) { asm volatile ( @@ -325,6 +341,7 @@ * * @return Current value of TPR. - */ -static inline uint64_t tpr_read(void) + * + */ +NO_TRACE static inline uint64_t tpr_read(void) { uint64_t v; @@ -341,6 +358,7 @@ * * @param v New value of TPR. - */ -static inline void tpr_write(uint64_t v) + * + */ +NO_TRACE static inline void tpr_write(uint64_t v) { asm volatile ( @@ -356,6 +374,7 @@ * * @return Old interrupt priority level. - */ -static ipl_t interrupts_disable(void) + * + */ +NO_TRACE static ipl_t interrupts_disable(void) { uint64_t v; @@ -377,6 +396,7 @@ * * @return Old interrupt priority level. - */ -static ipl_t interrupts_enable(void) + * + */ +NO_TRACE static ipl_t interrupts_enable(void) { uint64_t v; @@ -399,6 +419,7 @@ * * @param ipl Saved interrupt priority level. - */ -static inline void interrupts_restore(ipl_t ipl) + * + */ +NO_TRACE static inline void interrupts_restore(ipl_t ipl) { if (ipl & PSR_I_MASK) @@ -411,6 +432,7 @@ * * @return PSR. - */ -static inline ipl_t interrupts_read(void) + * + */ +NO_TRACE static inline ipl_t interrupts_read(void) { return (ipl_t) psr_read(); @@ -422,5 +444,5 @@ * */ -static inline bool interrupts_disabled(void) +NO_TRACE static inline bool interrupts_disabled(void) { return !(psr_read() & PSR_I_MASK); @@ -428,5 +450,5 @@ /** Disable protection key checking. */ -static inline void pk_disable(void) +NO_TRACE static inline void pk_disable(void) { asm volatile ( Index: kernel/arch/ia64/include/atomic.h =================================================================== --- kernel/arch/ia64/include/atomic.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ia64/include/atomic.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -36,5 +36,7 @@ #define KERN_ia64_ATOMIC_H_ -static inline atomic_count_t test_and_set(atomic_t *val) +#include + +NO_TRACE static inline atomic_count_t test_and_set(atomic_t *val) { atomic_count_t v; @@ -50,5 +52,5 @@ } -static inline void atomic_lock_arch(atomic_t *val) +NO_TRACE static inline void atomic_lock_arch(atomic_t *val) { do { @@ -57,5 +59,5 @@ } -static inline void atomic_inc(atomic_t *val) +NO_TRACE static inline void atomic_inc(atomic_t *val) { atomic_count_t v; @@ -68,5 +70,5 @@ } -static inline void atomic_dec(atomic_t *val) +NO_TRACE static inline void atomic_dec(atomic_t *val) { atomic_count_t v; @@ -79,5 +81,5 @@ } -static inline atomic_count_t atomic_preinc(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_preinc(atomic_t *val) { atomic_count_t v; @@ -92,5 +94,5 @@ } -static inline atomic_count_t atomic_predec(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_predec(atomic_t *val) { atomic_count_t v; @@ -105,5 +107,5 @@ } -static inline atomic_count_t atomic_postinc(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) { atomic_count_t v; @@ -118,5 +120,5 @@ } -static inline atomic_count_t atomic_postdec(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) { atomic_count_t v; Index: kernel/arch/ia64/include/cpu.h =================================================================== --- kernel/arch/ia64/include/cpu.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ia64/include/cpu.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -40,7 +40,13 @@ #include #include +#include -#define FAMILY_ITANIUM 0x7 -#define FAMILY_ITANIUM2 0x1f +#define FAMILY_ITANIUM 0x7 +#define FAMILY_ITANIUM2 0x1f + +#define CR64_ID_SHIFT 24 +#define CR64_ID_MASK 0xff000000 +#define CR64_EID_SHIFT 16 +#define CR64_EID_MASK 0xff0000 typedef struct { @@ -55,38 +61,35 @@ * * @return Value of CPUID[n] register. + * */ -static inline uint64_t cpuid_read(int n) +NO_TRACE static inline uint64_t cpuid_read(int n) { uint64_t v; - asm volatile ("mov %0 = cpuid[%1]\n" : "=r" (v) : "r" (n)); + asm volatile ( + "mov %[v] = cpuid[%[r]]\n" + : [v] "=r" (v) + : [r] "r" (n) + ); return v; } - -#define CR64_ID_SHIFT 24 -#define CR64_ID_MASK 0xff000000 -#define CR64_EID_SHIFT 16 -#define CR64_EID_MASK 0xff0000 - -static inline int ia64_get_cpu_id(void) +NO_TRACE static inline int ia64_get_cpu_id(void) { - uint64_t cr64=cr64_read(); - return ((CR64_ID_MASK)&cr64)>>CR64_ID_SHIFT; + uint64_t cr64 = cr64_read(); + return ((CR64_ID_MASK) &cr64) >> CR64_ID_SHIFT; } -static inline int ia64_get_cpu_eid(void) +NO_TRACE static inline int ia64_get_cpu_eid(void) { - uint64_t cr64=cr64_read(); - return ((CR64_EID_MASK)&cr64)>>CR64_EID_SHIFT; + uint64_t cr64 = cr64_read(); + return ((CR64_EID_MASK) &cr64) >> CR64_EID_SHIFT; } - -static inline void ipi_send_ipi(int id, int eid, int intno) +NO_TRACE static inline void ipi_send_ipi(int id, int eid, int intno) { (bootinfo->sapic)[2 * (id * 256 + eid)] = intno; srlz_d(); - } Index: kernel/arch/ia64/include/cycle.h =================================================================== --- kernel/arch/ia64/include/cycle.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ia64/include/cycle.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -36,5 +36,7 @@ #define KERN_ia64_CYCLE_H_ -static inline uint64_t get_cycle(void) +#include + +NO_TRACE static inline uint64_t get_cycle(void) { return 0; Index: kernel/arch/ia64/include/interrupt.h =================================================================== --- kernel/arch/ia64/include/interrupt.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ia64/include/interrupt.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -38,4 +38,5 @@ #include #include +#include /** ia64 has 256 INRs. */ @@ -133,5 +134,8 @@ } istate_t; -static inline void istate_set_retaddr(istate_t *istate, uintptr_t retaddr) +extern void *ivt; + +NO_TRACE static inline void istate_set_retaddr(istate_t *istate, + uintptr_t retaddr) { istate->cr_iip = retaddr; @@ -139,29 +143,29 @@ } -static inline unative_t istate_get_pc(istate_t *istate) +NO_TRACE static inline unative_t istate_get_pc(istate_t *istate) { return istate->cr_iip; } -static inline unative_t istate_get_fp(istate_t *istate) +NO_TRACE static inline unative_t istate_get_fp(istate_t *istate) { - return 0; /* FIXME */ + /* FIXME */ + + return 0; } -static inline int istate_from_uspace(istate_t *istate) +NO_TRACE static inline int istate_from_uspace(istate_t *istate) { return (istate->cr_iip) < 0xe000000000000000ULL; } -extern void *ivt; +extern void general_exception(uint64_t, istate_t *); +extern int break_instruction(uint64_t, istate_t *); +extern void universal_handler(uint64_t, istate_t *); +extern void nop_handler(uint64_t, istate_t *); +extern void external_interrupt(uint64_t, istate_t *); +extern void disabled_fp_register(uint64_t, istate_t *); -extern void general_exception(uint64_t vector, istate_t *istate); -extern int break_instruction(uint64_t vector, istate_t *istate); -extern void universal_handler(uint64_t vector, istate_t *istate); -extern void nop_handler(uint64_t vector, istate_t *istate); -extern void external_interrupt(uint64_t vector, istate_t *istate); -extern void disabled_fp_register(uint64_t vector, istate_t *istate); - -extern void trap_virtual_enable_irqs(uint16_t irqmask); +extern void trap_virtual_enable_irqs(uint16_t); #endif Index: kernel/arch/ia64/include/mm/page.h =================================================================== --- kernel/arch/ia64/include/mm/page.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ia64/include/mm/page.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -208,5 +208,5 @@ * @return Address of the head of VHPT collision chain. */ -static inline uint64_t thash(uint64_t va) +NO_TRACE static inline uint64_t thash(uint64_t va) { uint64_t ret; @@ -230,5 +230,5 @@ * @return The unique tag for VPN and RID in the collision chain returned by thash(). */ -static inline uint64_t ttag(uint64_t va) +NO_TRACE static inline uint64_t ttag(uint64_t va) { uint64_t ret; @@ -249,5 +249,5 @@ * @return Current contents of rr[i]. */ -static inline uint64_t rr_read(size_t i) +NO_TRACE static inline uint64_t rr_read(size_t i) { uint64_t ret; @@ -269,5 +269,5 @@ * @param v Value to be written to rr[i]. */ -static inline void rr_write(size_t i, uint64_t v) +NO_TRACE static inline void rr_write(size_t i, uint64_t v) { ASSERT(i < REGION_REGISTERS); @@ -284,5 +284,5 @@ * @return Current value stored in PTA. */ -static inline uint64_t pta_read(void) +NO_TRACE static inline uint64_t pta_read(void) { uint64_t ret; @@ -300,5 +300,5 @@ * @param v New value to be stored in PTA. */ -static inline void pta_write(uint64_t v) +NO_TRACE static inline void pta_write(uint64_t v) { asm volatile ( Index: kernel/arch/mips32/include/asm.h =================================================================== --- kernel/arch/mips32/include/asm.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/mips32/include/asm.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -38,9 +38,16 @@ #include #include +#include -static inline void cpu_sleep(void) +NO_TRACE static inline void cpu_sleep(void) { - /* Most of the simulators do not support */ -/* asm volatile ("wait"); */ + /* + * Unfortunatelly most of the simulators do not support + * + * asm volatile ( + * "wait" + * ); + * + */ } @@ -52,5 +59,5 @@ * */ -static inline uintptr_t get_stack_base(void) +NO_TRACE static inline uintptr_t get_stack_base(void) { uintptr_t base; @@ -65,44 +72,43 @@ } -extern void cpu_halt(void) __attribute__((noreturn)); -extern void asm_delay_loop(uint32_t t); -extern void userspace_asm(uintptr_t ustack, uintptr_t uspace_uarg, - uintptr_t entry); - -extern ipl_t interrupts_disable(void); -extern ipl_t interrupts_enable(void); -extern void interrupts_restore(ipl_t ipl); -extern ipl_t interrupts_read(void); -extern bool interrupts_disabled(void); - -static inline void pio_write_8(ioport8_t *port, uint8_t v) +NO_TRACE static inline void pio_write_8(ioport8_t *port, uint8_t v) { *port = v; } -static inline void pio_write_16(ioport16_t *port, uint16_t v) +NO_TRACE static inline void pio_write_16(ioport16_t *port, uint16_t v) { *port = v; } -static inline void pio_write_32(ioport32_t *port, uint32_t v) +NO_TRACE static inline void pio_write_32(ioport32_t *port, uint32_t v) { *port = v; } -static inline uint8_t pio_read_8(ioport8_t *port) +NO_TRACE static inline uint8_t pio_read_8(ioport8_t *port) { return *port; } -static inline uint16_t pio_read_16(ioport16_t *port) +NO_TRACE static inline uint16_t pio_read_16(ioport16_t *port) { return *port; } -static inline uint32_t pio_read_32(ioport32_t *port) +NO_TRACE static inline uint32_t pio_read_32(ioport32_t *port) { return *port; } + +extern void cpu_halt(void) __attribute__((noreturn)); +extern void asm_delay_loop(uint32_t); +extern void userspace_asm(uintptr_t, uintptr_t, uintptr_t); + +extern ipl_t interrupts_disable(void); +extern ipl_t interrupts_enable(void); +extern void interrupts_restore(ipl_t); +extern ipl_t interrupts_read(void); +extern bool interrupts_disabled(void); #endif Index: kernel/arch/mips32/include/atomic.h =================================================================== --- kernel/arch/mips32/include/atomic.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/mips32/include/atomic.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -36,4 +36,6 @@ #define KERN_mips32_ATOMIC_H_ +#include + #define atomic_inc(x) ((void) atomic_add(x, 1)) #define atomic_dec(x) ((void) atomic_add(x, -1)) @@ -53,5 +55,6 @@ * */ -static inline atomic_count_t atomic_add(atomic_t *val, atomic_count_t i) +NO_TRACE static inline atomic_count_t atomic_add(atomic_t *val, + atomic_count_t i) { atomic_count_t tmp; @@ -76,5 +79,5 @@ } -static inline atomic_count_t test_and_set(atomic_t *val) +NO_TRACE static inline atomic_count_t test_and_set(atomic_t *val) { atomic_count_t tmp; @@ -98,5 +101,5 @@ } -static inline void atomic_lock_arch(atomic_t *val) +NO_TRACE static inline void atomic_lock_arch(atomic_t *val) { do { Index: kernel/arch/mips32/include/barrier.h =================================================================== --- kernel/arch/mips32/include/barrier.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/mips32/include/barrier.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup mips32 +/** @addtogroup mips32 * @{ */ @@ -39,10 +39,10 @@ * TODO: implement true MIPS memory barriers for macros below. */ -#define CS_ENTER_BARRIER() asm volatile ("" ::: "memory") -#define CS_LEAVE_BARRIER() asm volatile ("" ::: "memory") +#define CS_ENTER_BARRIER() asm volatile ("" ::: "memory") +#define CS_LEAVE_BARRIER() asm volatile ("" ::: "memory") -#define memory_barrier() asm volatile ("" ::: "memory") -#define read_barrier() asm volatile ("" ::: "memory") -#define write_barrier() asm volatile ("" ::: "memory") +#define memory_barrier() asm volatile ("" ::: "memory") +#define read_barrier() asm volatile ("" ::: "memory") +#define write_barrier() asm volatile ("" ::: "memory") #define smc_coherence(a) Index: kernel/arch/mips32/include/cycle.h =================================================================== --- kernel/arch/mips32/include/cycle.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/mips32/include/cycle.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -38,6 +38,7 @@ #include #include +#include -static inline uint64_t get_cycle(void) +NO_TRACE static inline uint64_t get_cycle(void) { return ((uint64_t) count_hi << 32) + ((uint64_t) cp0_count_read()); Index: kernel/arch/mips32/include/exception.h =================================================================== --- kernel/arch/mips32/include/exception.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/mips32/include/exception.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup mips32 +/** @addtogroup mips32 * @{ */ @@ -38,23 +38,24 @@ #include #include +#include -#define EXC_Int 0 -#define EXC_Mod 1 -#define EXC_TLBL 2 -#define EXC_TLBS 3 -#define EXC_AdEL 4 -#define EXC_AdES 5 -#define EXC_IBE 6 -#define EXC_DBE 7 -#define EXC_Sys 8 -#define EXC_Bp 9 -#define EXC_RI 10 -#define EXC_CpU 11 -#define EXC_Ov 12 -#define EXC_Tr 13 -#define EXC_VCEI 14 -#define EXC_FPE 15 -#define EXC_WATCH 23 -#define EXC_VCED 31 +#define EXC_Int 0 +#define EXC_Mod 1 +#define EXC_TLBL 2 +#define EXC_TLBS 3 +#define EXC_AdEL 4 +#define EXC_AdES 5 +#define EXC_IBE 6 +#define EXC_DBE 7 +#define EXC_Sys 8 +#define EXC_Bp 9 +#define EXC_RI 10 +#define EXC_CpU 11 +#define EXC_Ov 12 +#define EXC_Tr 13 +#define EXC_VCEI 14 +#define EXC_FPE 15 +#define EXC_WATCH 23 +#define EXC_VCED 31 typedef struct istate { @@ -82,11 +83,12 @@ uint32_t lo; uint32_t hi; - - uint32_t status; /* cp0_status */ - uint32_t epc; /* cp0_epc */ - uint32_t k1; /* We use it as thread-local pointer */ + + uint32_t status; /* cp0_status */ + uint32_t epc; /* cp0_epc */ + uint32_t k1; /* We use it as thread-local pointer */ } istate_t; -static inline void istate_set_retaddr(istate_t *istate, uintptr_t retaddr) +NO_TRACE static inline void istate_set_retaddr(istate_t *istate, + uintptr_t retaddr) { istate->epc = retaddr; @@ -94,15 +96,19 @@ /** Return true if exception happened while in userspace */ -static inline int istate_from_uspace(istate_t *istate) +NO_TRACE static inline int istate_from_uspace(istate_t *istate) { return istate->status & cp0_status_um_bit; } -static inline unative_t istate_get_pc(istate_t *istate) + +NO_TRACE static inline unative_t istate_get_pc(istate_t *istate) { return istate->epc; } -static inline unative_t istate_get_fp(istate_t *istate) + +NO_TRACE static inline unative_t istate_get_fp(istate_t *istate) { - return 0; /* FIXME */ + /* FIXME */ + + return 0; } Index: kernel/arch/mips32/include/faddr.h =================================================================== --- kernel/arch/mips32/include/faddr.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/mips32/include/faddr.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup mips32 +/** @addtogroup mips32 * @{ */ @@ -38,5 +38,5 @@ #include -#define FADDR(fptr) ((uintptr_t) (fptr)) +#define FADDR(fptr) ((uintptr_t) (fptr)) #endif Index: kernel/arch/mips32/include/mm/page.h =================================================================== --- kernel/arch/mips32/include/mm/page.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/mips32/include/mm/page.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -37,4 +37,5 @@ #include +#include #define PAGE_WIDTH FRAME_WIDTH @@ -155,5 +156,5 @@ -static inline unsigned int get_pt_flags(pte_t *pt, size_t i) +NO_TRACE static inline unsigned int get_pt_flags(pte_t *pt, size_t i) { pte_t *p = &pt[i]; @@ -168,5 +169,5 @@ } -static inline void set_pt_flags(pte_t *pt, size_t i, int flags) +NO_TRACE static inline void set_pt_flags(pte_t *pt, size_t i, int flags) { pte_t *p = &pt[i]; Index: kernel/arch/mips32/include/mm/tlb.h =================================================================== --- kernel/arch/mips32/include/mm/tlb.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/mips32/include/mm/tlb.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -39,4 +39,5 @@ #include #include +#include #define TLB_ENTRY_COUNT 48 @@ -126,5 +127,5 @@ * Probe TLB for Matching Entry. */ -static inline void tlbp(void) +NO_TRACE static inline void tlbp(void) { asm volatile ("tlbp\n\t"); @@ -136,5 +137,5 @@ * Read Indexed TLB Entry. */ -static inline void tlbr(void) +NO_TRACE static inline void tlbr(void) { asm volatile ("tlbr\n\t"); @@ -145,5 +146,5 @@ * Write Indexed TLB Entry. */ -static inline void tlbwi(void) +NO_TRACE static inline void tlbwi(void) { asm volatile ("tlbwi\n\t"); @@ -154,5 +155,5 @@ * Write Random TLB Entry. */ -static inline void tlbwr(void) +NO_TRACE static inline void tlbwr(void) { asm volatile ("tlbwr\n\t"); Index: kernel/arch/ppc32/include/asm.h =================================================================== --- kernel/arch/ppc32/include/asm.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ppc32/include/asm.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -40,6 +40,7 @@ #include #include - -static inline uint32_t msr_read(void) +#include + +NO_TRACE static inline uint32_t msr_read(void) { uint32_t msr; @@ -53,5 +54,5 @@ } -static inline void msr_write(uint32_t msr) +NO_TRACE static inline void msr_write(uint32_t msr) { asm volatile ( @@ -61,5 +62,5 @@ } -static inline void sr_set(uint32_t flags, asid_t asid, uint32_t sr) +NO_TRACE static inline void sr_set(uint32_t flags, asid_t asid, uint32_t sr) { asm volatile ( @@ -70,5 +71,5 @@ } -static inline uint32_t sr_get(uint32_t vaddr) +NO_TRACE static inline uint32_t sr_get(uint32_t vaddr) { uint32_t vsid; @@ -83,5 +84,5 @@ } -static inline uint32_t sdr1_get(void) +NO_TRACE static inline uint32_t sdr1_get(void) { uint32_t sdr1; @@ -103,5 +104,5 @@ * */ -static inline ipl_t interrupts_enable(void) +NO_TRACE static inline ipl_t interrupts_enable(void) { ipl_t ipl = msr_read(); @@ -118,5 +119,5 @@ * */ -static inline ipl_t interrupts_disable(void) +NO_TRACE static inline ipl_t interrupts_disable(void) { ipl_t ipl = msr_read(); @@ -132,5 +133,5 @@ * */ -static inline void interrupts_restore(ipl_t ipl) +NO_TRACE static inline void interrupts_restore(ipl_t ipl) { msr_write((msr_read() & (~MSR_EE)) | (ipl & MSR_EE)); @@ -144,5 +145,5 @@ * */ -static inline ipl_t interrupts_read(void) +NO_TRACE static inline ipl_t interrupts_read(void) { return msr_read(); @@ -154,5 +155,5 @@ * */ -static inline bool interrupts_disabled(void) +NO_TRACE static inline bool interrupts_disabled(void) { return ((msr_read() & MSR_EE) == 0); @@ -166,5 +167,5 @@ * */ -static inline uintptr_t get_stack_base(void) +NO_TRACE static inline uintptr_t get_stack_base(void) { uintptr_t base; @@ -179,6 +180,36 @@ } -static inline void cpu_sleep(void) -{ +NO_TRACE static inline void cpu_sleep(void) +{ +} + +NO_TRACE static inline void pio_write_8(ioport8_t *port, uint8_t v) +{ + *port = v; +} + +NO_TRACE static inline void pio_write_16(ioport16_t *port, uint16_t v) +{ + *port = v; +} + +NO_TRACE static inline void pio_write_32(ioport32_t *port, uint32_t v) +{ + *port = v; +} + +NO_TRACE static inline uint8_t pio_read_8(ioport8_t *port) +{ + return *port; +} + +NO_TRACE static inline uint16_t pio_read_16(ioport16_t *port) +{ + return *port; +} + +NO_TRACE static inline uint32_t pio_read_32(ioport32_t *port) +{ + return *port; } @@ -187,34 +218,4 @@ extern void userspace_asm(uintptr_t uspace_uarg, uintptr_t stack, uintptr_t entry); -static inline void pio_write_8(ioport8_t *port, uint8_t v) -{ - *port = v; -} - -static inline void pio_write_16(ioport16_t *port, uint16_t v) -{ - *port = v; -} - -static inline void pio_write_32(ioport32_t *port, uint32_t v) -{ - *port = v; -} - -static inline uint8_t pio_read_8(ioport8_t *port) -{ - return *port; -} - -static inline uint16_t pio_read_16(ioport16_t *port) -{ - return *port; -} - -static inline uint32_t pio_read_32(ioport32_t *port) -{ - return *port; -} - #endif Index: kernel/arch/ppc32/include/atomic.h =================================================================== --- kernel/arch/ppc32/include/atomic.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ppc32/include/atomic.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -36,5 +36,7 @@ #define KERN_ppc32_ATOMIC_H_ -static inline void atomic_inc(atomic_t *val) +#include + +NO_TRACE static inline void atomic_inc(atomic_t *val) { atomic_count_t tmp; @@ -54,5 +56,5 @@ } -static inline void atomic_dec(atomic_t *val) +NO_TRACE static inline void atomic_dec(atomic_t *val) { atomic_count_t tmp; @@ -72,5 +74,5 @@ } -static inline atomic_count_t atomic_postinc(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) { atomic_inc(val); @@ -78,5 +80,5 @@ } -static inline atomic_count_t atomic_postdec(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) { atomic_dec(val); @@ -84,5 +86,5 @@ } -static inline atomic_count_t atomic_preinc(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_preinc(atomic_t *val) { atomic_inc(val); @@ -90,5 +92,5 @@ } -static inline atomic_count_t atomic_predec(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_predec(atomic_t *val) { atomic_dec(val); Index: kernel/arch/ppc32/include/barrier.h =================================================================== --- kernel/arch/ppc32/include/barrier.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ppc32/include/barrier.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -36,4 +36,6 @@ #define KERN_ppc32_BARRIER_H_ +#include + #define CS_ENTER_BARRIER() asm volatile ("" ::: "memory") #define CS_LEAVE_BARRIER() asm volatile ("" ::: "memory") @@ -58,5 +60,5 @@ */ -static inline void smc_coherence(void *addr) +NO_TRACE static inline void smc_coherence(void *addr) { asm volatile ( @@ -70,5 +72,5 @@ } -static inline void smc_coherence_block(void *addr, unsigned int len) +NO_TRACE static inline void smc_coherence_block(void *addr, unsigned int len) { unsigned int i; Index: kernel/arch/ppc32/include/cpu.h =================================================================== --- kernel/arch/ppc32/include/cpu.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ppc32/include/cpu.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -52,4 +52,5 @@ #include +#include typedef struct { @@ -58,5 +59,5 @@ } __attribute__ ((packed)) cpu_arch_t; -static inline void cpu_version(cpu_arch_t *info) +NO_TRACE static inline void cpu_version(cpu_arch_t *info) { asm volatile ( Index: kernel/arch/ppc32/include/cycle.h =================================================================== --- kernel/arch/ppc32/include/cycle.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ppc32/include/cycle.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -36,5 +36,7 @@ #define KERN_ppc32_CYCLE_H_ -static inline uint64_t get_cycle(void) +#include + +NO_TRACE static inline uint64_t get_cycle(void) { uint32_t lower; Index: kernel/arch/ppc32/include/exception.h =================================================================== --- kernel/arch/ppc32/include/exception.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ppc32/include/exception.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -38,4 +38,5 @@ #include #include +#include typedef struct istate { @@ -81,5 +82,6 @@ } istate_t; -static inline void istate_set_retaddr(istate_t *istate, uintptr_t retaddr) +NO_TRACE static inline void istate_set_retaddr(istate_t *istate, + uintptr_t retaddr) { istate->pc = retaddr; @@ -91,15 +93,15 @@ * */ -static inline int istate_from_uspace(istate_t *istate) +NO_TRACE static inline int istate_from_uspace(istate_t *istate) { return (istate->srr1 & MSR_PR) != 0; } -static inline unative_t istate_get_pc(istate_t *istate) +NO_TRACE static inline unative_t istate_get_pc(istate_t *istate) { return istate->pc; } -static inline unative_t istate_get_fp(istate_t *istate) +NO_TRACE static inline unative_t istate_get_fp(istate_t *istate) { return istate->sp; Index: kernel/arch/ppc32/include/mm/frame.h =================================================================== --- kernel/arch/ppc32/include/mm/frame.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ppc32/include/mm/frame.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -43,8 +43,9 @@ #include +#include extern uintptr_t last_frame; -static inline uint32_t physmem_top(void) +NO_TRACE static inline uint32_t physmem_top(void) { uint32_t physmem; Index: kernel/arch/ppc32/include/mm/page.h =================================================================== --- kernel/arch/ppc32/include/mm/page.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/ppc32/include/mm/page.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -37,4 +37,5 @@ #include +#include #define PAGE_WIDTH FRAME_WIDTH @@ -153,5 +154,5 @@ } pte_t; -static inline unsigned int get_pt_flags(pte_t *pt, size_t i) +NO_TRACE static inline unsigned int get_pt_flags(pte_t *pt, size_t i) { pte_t *entry = &pt[i]; @@ -166,5 +167,5 @@ } -static inline void set_pt_flags(pte_t *pt, size_t i, int flags) +NO_TRACE static inline void set_pt_flags(pte_t *pt, size_t i, int flags) { pte_t *entry = &pt[i]; Index: kernel/arch/sparc64/include/asm.h =================================================================== --- kernel/arch/sparc64/include/asm.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/asm.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup sparc64 +/** @addtogroup sparc64 * @{ */ @@ -43,6 +43,7 @@ #include #include - -static inline void pio_write_8(ioport8_t *port, uint8_t v) +#include + +NO_TRACE static inline void pio_write_8(ioport8_t *port, uint8_t v) { *port = v; @@ -50,5 +51,5 @@ } -static inline void pio_write_16(ioport16_t *port, uint16_t v) +NO_TRACE static inline void pio_write_16(ioport16_t *port, uint16_t v) { *port = v; @@ -56,5 +57,5 @@ } -static inline void pio_write_32(ioport32_t *port, uint32_t v) +NO_TRACE static inline void pio_write_32(ioport32_t *port, uint32_t v) { *port = v; @@ -62,31 +63,22 @@ } -static inline uint8_t pio_read_8(ioport8_t *port) -{ - uint8_t rv; - - rv = *port; +NO_TRACE static inline uint8_t pio_read_8(ioport8_t *port) +{ + uint8_t rv = *port; memory_barrier(); - return rv; } -static inline uint16_t pio_read_16(ioport16_t *port) -{ - uint16_t rv; - - rv = *port; +NO_TRACE static inline uint16_t pio_read_16(ioport16_t *port) +{ + uint16_t rv = *port; memory_barrier(); - return rv; } -static inline uint32_t pio_read_32(ioport32_t *port) -{ - uint32_t rv; - - rv = *port; +NO_TRACE static inline uint32_t pio_read_32(ioport32_t *port) +{ + uint32_t rv = *port; memory_barrier(); - return rv; } @@ -95,10 +87,14 @@ * * @return Value of PSTATE register. - */ -static inline uint64_t pstate_read(void) -{ - uint64_t v; - - asm volatile ("rdpr %%pstate, %0\n" : "=r" (v)); + * + */ +NO_TRACE static inline uint64_t pstate_read(void) +{ + uint64_t v; + + asm volatile ( + "rdpr %%pstate, %[v]\n" + : [v] "=r" (v) + ); return v; @@ -108,8 +104,13 @@ * * @param v New value of PSTATE register. - */ -static inline void pstate_write(uint64_t v) -{ - asm volatile ("wrpr %0, %1, %%pstate\n" : : "r" (v), "i" (0)); + * + */ +NO_TRACE static inline void pstate_write(uint64_t v) +{ + asm volatile ( + "wrpr %[v], %[zero], %%pstate\n" + :: [v] "r" (v), + [zero] "i" (0) + ); } @@ -117,10 +118,14 @@ * * @return Value of TICK_comapre register. - */ -static inline uint64_t tick_compare_read(void) -{ - uint64_t v; - - asm volatile ("rd %%tick_cmpr, %0\n" : "=r" (v)); + * + */ +NO_TRACE static inline uint64_t tick_compare_read(void) +{ + uint64_t v; + + asm volatile ( + "rd %%tick_cmpr, %[v]\n" + : [v] "=r" (v) + ); return v; @@ -130,8 +135,13 @@ * * @param v New value of TICK_comapre register. - */ -static inline void tick_compare_write(uint64_t v) -{ - asm volatile ("wr %0, %1, %%tick_cmpr\n" : : "r" (v), "i" (0)); + * + */ +NO_TRACE static inline void tick_compare_write(uint64_t v) +{ + asm volatile ( + "wr %[v], %[zero], %%tick_cmpr\n" + :: [v] "r" (v), + [zero] "i" (0) + ); } @@ -139,10 +149,14 @@ * * @return Value of STICK_compare register. - */ -static inline uint64_t stick_compare_read(void) -{ - uint64_t v; - - asm volatile ("rd %%asr25, %0\n" : "=r" (v)); + * + */ +NO_TRACE static inline uint64_t stick_compare_read(void) +{ + uint64_t v; + + asm volatile ( + "rd %%asr25, %[v]\n" + : [v] "=r" (v) + ); return v; @@ -152,8 +166,13 @@ * * @param v New value of STICK_comapre register. - */ -static inline void stick_compare_write(uint64_t v) -{ - asm volatile ("wr %0, %1, %%asr25\n" : : "r" (v), "i" (0)); + * + */ +NO_TRACE static inline void stick_compare_write(uint64_t v) +{ + asm volatile ( + "wr %[v], %[zero], %%asr25\n" + :: [v] "r" (v), + [zero] "i" (0) + ); } @@ -161,10 +180,14 @@ * * @return Value of TICK register. - */ -static inline uint64_t tick_read(void) -{ - uint64_t v; - - asm volatile ("rdpr %%tick, %0\n" : "=r" (v)); + * + */ +NO_TRACE static inline uint64_t tick_read(void) +{ + uint64_t v; + + asm volatile ( + "rdpr %%tick, %[v]\n" + : [v] "=r" (v) + ); return v; @@ -174,8 +197,13 @@ * * @param v New value of TICK register. - */ -static inline void tick_write(uint64_t v) -{ - asm volatile ("wrpr %0, %1, %%tick\n" : : "r" (v), "i" (0)); + * + */ +NO_TRACE static inline void tick_write(uint64_t v) +{ + asm volatile ( + "wrpr %[v], %[zero], %%tick\n" + :: [v] "r" (v), + [zero] "i" (0) + ); } @@ -183,10 +211,14 @@ * * @return Value of FPRS register. - */ -static inline uint64_t fprs_read(void) -{ - uint64_t v; - - asm volatile ("rd %%fprs, %0\n" : "=r" (v)); + * + */ +NO_TRACE static inline uint64_t fprs_read(void) +{ + uint64_t v; + + asm volatile ( + "rd %%fprs, %[v]\n" + : [v] "=r" (v) + ); return v; @@ -196,8 +228,13 @@ * * @param v New value of FPRS register. - */ -static inline void fprs_write(uint64_t v) -{ - asm volatile ("wr %0, %1, %%fprs\n" : : "r" (v), "i" (0)); + * + */ +NO_TRACE static inline void fprs_write(uint64_t v) +{ + asm volatile ( + "wr %[v], %[zero], %%fprs\n" + :: [v] "r" (v), + [zero] "i" (0) + ); } @@ -205,11 +242,15 @@ * * @return Value of SOFTINT register. - */ -static inline uint64_t softint_read(void) -{ - uint64_t v; - - asm volatile ("rd %%softint, %0\n" : "=r" (v)); - + * + */ +NO_TRACE static inline uint64_t softint_read(void) +{ + uint64_t v; + + asm volatile ( + "rd %%softint, %[v]\n" + : [v] "=r" (v) + ); + return v; } @@ -218,8 +259,13 @@ * * @param v New value of SOFTINT register. - */ -static inline void softint_write(uint64_t v) -{ - asm volatile ("wr %0, %1, %%softint\n" : : "r" (v), "i" (0)); + * + */ +NO_TRACE static inline void softint_write(uint64_t v) +{ + asm volatile ( + "wr %[v], %[zero], %%softint\n" + :: [v] "r" (v), + [zero] "i" (0) + ); } @@ -229,8 +275,13 @@ * * @param v New value of CLEAR_SOFTINT register. - */ -static inline void clear_softint_write(uint64_t v) -{ - asm volatile ("wr %0, %1, %%clear_softint\n" : : "r" (v), "i" (0)); + * + */ +NO_TRACE static inline void clear_softint_write(uint64_t v) +{ + asm volatile ( + "wr %[v], %[zero], %%clear_softint\n" + :: [v] "r" (v), + [zero] "i" (0) + ); } @@ -240,8 +291,13 @@ * * @param v New value of SET_SOFTINT register. - */ -static inline void set_softint_write(uint64_t v) -{ - asm volatile ("wr %0, %1, %%set_softint\n" : : "r" (v), "i" (0)); + * + */ +NO_TRACE static inline void set_softint_write(uint64_t v) +{ + asm volatile ( + "wr %[v], %[zero], %%set_softint\n" + :: [v] "r" (v), + [zero] "i" (0) + ); } @@ -252,10 +308,10 @@ * * @return Old interrupt priority level. - */ -static inline ipl_t interrupts_enable(void) { + * + */ +NO_TRACE static inline ipl_t interrupts_enable(void) { pstate_reg_t pstate; - uint64_t value; - - value = pstate_read(); + uint64_t value = pstate_read(); + pstate.value = value; pstate.ie = true; @@ -271,10 +327,10 @@ * * @return Old interrupt priority level. - */ -static inline ipl_t interrupts_disable(void) { + * + */ +NO_TRACE static inline ipl_t interrupts_disable(void) { pstate_reg_t pstate; - uint64_t value; - - value = pstate_read(); + uint64_t value = pstate_read(); + pstate.value = value; pstate.ie = false; @@ -289,6 +345,7 @@ * * @param ipl Saved interrupt priority level. - */ -static inline void interrupts_restore(ipl_t ipl) { + * + */ +NO_TRACE static inline void interrupts_restore(ipl_t ipl) { pstate_reg_t pstate; @@ -303,6 +360,7 @@ * * @return Current interrupt priority level. - */ -static inline ipl_t interrupts_read(void) { + * + */ +NO_TRACE static inline ipl_t interrupts_read(void) { return (ipl_t) pstate_read(); } @@ -313,8 +371,8 @@ * */ -static inline bool interrupts_disabled(void) +NO_TRACE static inline bool interrupts_disabled(void) { pstate_reg_t pstate; - + pstate.value = pstate_read(); return !pstate.ie; @@ -326,10 +384,15 @@ * The stack is assumed to be STACK_SIZE bytes long. * The stack must start on page boundary. - */ -static inline uintptr_t get_stack_base(void) + * + */ +NO_TRACE static inline uintptr_t get_stack_base(void) { uintptr_t unbiased_sp; - asm volatile ("add %%sp, %1, %0\n" : "=r" (unbiased_sp) : "i" (STACK_BIAS)); + asm volatile ( + "add %%sp, %[stack_bias], %[unbiased_sp]\n" + : [unbiased_sp] "=r" (unbiased_sp) + : [stack_bias] "i" (STACK_BIAS) + ); return ALIGN_DOWN(unbiased_sp, STACK_SIZE); @@ -339,10 +402,14 @@ * * @return Value of VER register. - */ -static inline uint64_t ver_read(void) -{ - uint64_t v; - - asm volatile ("rdpr %%ver, %0\n" : "=r" (v)); + * + */ +NO_TRACE static inline uint64_t ver_read(void) +{ + uint64_t v; + + asm volatile ( + "rdpr %%ver, %[v]\n" + : [v] "=r" (v) + ); return v; @@ -352,10 +419,14 @@ * * @return Current value in TPC. - */ -static inline uint64_t tpc_read(void) -{ - uint64_t v; - - asm volatile ("rdpr %%tpc, %0\n" : "=r" (v)); + * + */ +NO_TRACE static inline uint64_t tpc_read(void) +{ + uint64_t v; + + asm volatile ( + "rdpr %%tpc, %[v]\n" + : [v] "=r" (v) + ); return v; @@ -365,10 +436,14 @@ * * @return Current value in TL. - */ -static inline uint64_t tl_read(void) -{ - uint64_t v; - - asm volatile ("rdpr %%tl, %0\n" : "=r" (v)); + * + */ +NO_TRACE static inline uint64_t tl_read(void) +{ + uint64_t v; + + asm volatile ( + "rdpr %%tl, %[v]\n" + : [v] "=r" (v) + ); return v; @@ -378,10 +453,14 @@ * * @return Current value in TBA. - */ -static inline uint64_t tba_read(void) -{ - uint64_t v; - - asm volatile ("rdpr %%tba, %0\n" : "=r" (v)); + * + */ +NO_TRACE static inline uint64_t tba_read(void) +{ + uint64_t v; + + asm volatile ( + "rdpr %%tba, %[v]\n" + : [v] "=r" (v) + ); return v; @@ -391,8 +470,13 @@ * * @param v New value of TBA. - */ -static inline void tba_write(uint64_t v) -{ - asm volatile ("wrpr %0, %1, %%tba\n" : : "r" (v), "i" (0)); + * + */ +NO_TRACE static inline void tba_write(uint64_t v) +{ + asm volatile ( + "wrpr %[v], %[zero], %%tba\n" + :: [v] "r" (v), + [zero] "i" (0) + ); } @@ -400,13 +484,20 @@ * * @param asi ASI determining the alternate space. - * @param va Virtual address within the ASI. - * - * @return Value read from the virtual address in the specified address space. - */ -static inline uint64_t asi_u64_read(asi_t asi, uintptr_t va) -{ - uint64_t v; - - asm volatile ("ldxa [%1] %2, %0\n" : "=r" (v) : "r" (va), "i" ((unsigned) asi)); + * @param va Virtual address within the ASI. + * + * @return Value read from the virtual address in + * the specified address space. + * + */ +NO_TRACE static inline uint64_t asi_u64_read(asi_t asi, uintptr_t va) +{ + uint64_t v; + + asm volatile ( + "ldxa [%[va]] %[asi], %[v]\n" + : [v] "=r" (v) + : [va] "r" (va), + [asi] "i" ((unsigned int) asi) + ); return v; @@ -416,14 +507,21 @@ * * @param asi ASI determining the alternate space. - * @param va Virtual address within the ASI. - * @param v Value to be written. - */ -static inline void asi_u64_write(asi_t asi, uintptr_t va, uint64_t v) -{ - asm volatile ("stxa %0, [%1] %2\n" : : "r" (v), "r" (va), "i" ((unsigned) asi) : "memory"); + * @param va Virtual address within the ASI. + * @param v Value to be written. + * + */ +NO_TRACE static inline void asi_u64_write(asi_t asi, uintptr_t va, uint64_t v) +{ + asm volatile ( + "stxa %[v], [%[va]] %[asi]\n" + :: [v] "r" (v), + [va] "r" (va), + [asi] "i" ((unsigned int) asi) + : "memory" + ); } /** Flush all valid register windows to memory. */ -static inline void flushw(void) +NO_TRACE static inline void flushw(void) { asm volatile ("flushw\n"); @@ -431,5 +529,5 @@ /** Switch to nucleus by setting TL to 1. */ -static inline void nucleus_enter(void) +NO_TRACE static inline void nucleus_enter(void) { asm volatile ("wrpr %g0, 1, %tl\n"); @@ -437,5 +535,5 @@ /** Switch from nucleus by setting TL to 0. */ -static inline void nucleus_leave(void) +NO_TRACE static inline void nucleus_leave(void) { asm volatile ("wrpr %g0, %g0, %tl\n"); Index: kernel/arch/sparc64/include/atomic.h =================================================================== --- kernel/arch/sparc64/include/atomic.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/atomic.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -39,4 +39,5 @@ #include #include +#include /** Atomic add operation. @@ -50,5 +51,6 @@ * */ -static inline atomic_count_t atomic_add(atomic_t *val, atomic_count_t i) +NO_TRACE static inline atomic_count_t atomic_add(atomic_t *val, + atomic_count_t i) { atomic_count_t a; @@ -72,35 +74,35 @@ } -static inline atomic_count_t atomic_preinc(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_preinc(atomic_t *val) { return atomic_add(val, 1) + 1; } -static inline atomic_count_t atomic_postinc(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_postinc(atomic_t *val) { return atomic_add(val, 1); } -static inline atomic_count_t atomic_predec(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_predec(atomic_t *val) { return atomic_add(val, -1) - 1; } -static inline atomic_count_t atomic_postdec(atomic_t *val) +NO_TRACE static inline atomic_count_t atomic_postdec(atomic_t *val) { return atomic_add(val, -1); } -static inline void atomic_inc(atomic_t *val) +NO_TRACE static inline void atomic_inc(atomic_t *val) { (void) atomic_add(val, 1); } -static inline void atomic_dec(atomic_t *val) +NO_TRACE static inline void atomic_dec(atomic_t *val) { (void) atomic_add(val, -1); } -static inline atomic_count_t test_and_set(atomic_t *val) +NO_TRACE static inline atomic_count_t test_and_set(atomic_t *val) { atomic_count_t v = 1; @@ -117,5 +119,5 @@ } -static inline void atomic_lock_arch(atomic_t *val) +NO_TRACE static inline void atomic_lock_arch(atomic_t *val) { atomic_count_t tmp1 = 1; Index: kernel/arch/sparc64/include/barrier.h =================================================================== --- kernel/arch/sparc64/include/barrier.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/barrier.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup sparc64 +/** @addtogroup sparc64 * @{ */ @@ -36,8 +36,14 @@ #define KERN_sparc64_BARRIER_H_ +#include + #ifdef KERNEL + #include + #else + #include + #endif @@ -45,31 +51,47 @@ * Our critical section barriers are prepared for the weakest RMO memory model. */ -#define CS_ENTER_BARRIER() \ - asm volatile ( \ - "membar #LoadLoad | #LoadStore\n" \ - ::: "memory" \ - ) -#define CS_LEAVE_BARRIER() \ - asm volatile ( \ - "membar #StoreStore\n" \ - "membar #LoadStore\n" \ - ::: "memory" \ +#define CS_ENTER_BARRIER() \ + asm volatile ( \ + "membar #LoadLoad | #LoadStore\n" \ + ::: "memory" \ ) -#define memory_barrier() \ - asm volatile ("membar #LoadLoad | #StoreStore\n" ::: "memory") -#define read_barrier() \ - asm volatile ("membar #LoadLoad\n" ::: "memory") -#define write_barrier() \ - asm volatile ("membar #StoreStore\n" ::: "memory") +#define CS_LEAVE_BARRIER() \ + asm volatile ( \ + "membar #StoreStore\n" \ + "membar #LoadStore\n" \ + ::: "memory" \ + ) -#define flush(a) \ - asm volatile ("flush %0\n" :: "r" ((a)) : "memory") +#define memory_barrier() \ + asm volatile ( \ + "membar #LoadLoad | #StoreStore\n" \ + ::: "memory" \ + ) + +#define read_barrier() \ + asm volatile ( \ + "membar #LoadLoad\n" \ + ::: "memory" \ + ) + +#define write_barrier() \ + asm volatile ( \ + "membar #StoreStore\n" \ + ::: "memory" \ + ) + +#define flush(a) \ + asm volatile ( \ + "flush %[reg]\n" \ + :: [reg] "r" ((a)) \ + : "memory" \ + ) /** Flush Instruction pipeline. */ -static inline void flush_pipeline(void) +NO_TRACE static inline void flush_pipeline(void) { uint64_t pc; - + /* * The FLUSH instruction takes address parameter. @@ -80,51 +102,56 @@ * the %pc register will always be in the range mapped by * DTLB. + * */ - - asm volatile ( - "rd %%pc, %0\n" - "flush %0\n" - : "=&r" (pc) + + asm volatile ( + "rd %%pc, %[pc]\n" + "flush %[pc]\n" + : [pc] "=&r" (pc) ); } /** Memory Barrier instruction. */ -static inline void membar(void) +NO_TRACE static inline void membar(void) { - asm volatile ("membar #Sync\n"); + asm volatile ( + "membar #Sync\n" + ); } #if defined (US) -#define smc_coherence(a) \ -{ \ - write_barrier(); \ - flush((a)); \ -} +#define FLUSH_INVAL_MIN 4 -#define FLUSH_INVAL_MIN 4 -#define smc_coherence_block(a, l) \ -{ \ - unsigned long i; \ - write_barrier(); \ - for (i = 0; i < (l); i += FLUSH_INVAL_MIN) \ - flush((void *)(a) + i); \ -} +#define smc_coherence(a) \ + do { \ + write_barrier(); \ + flush((a)); \ + } while (0) + +#define smc_coherence_block(a, l) \ + do { \ + unsigned long i; \ + write_barrier(); \ + \ + for (i = 0; i < (l); i += FLUSH_INVAL_MIN) \ + flush((void *)(a) + i); \ + } while (0) #elif defined (US3) -#define smc_coherence(a) \ -{ \ - write_barrier(); \ - flush_pipeline(); \ -} +#define smc_coherence(a) \ + do { \ + write_barrier(); \ + flush_pipeline(); \ + } while (0) -#define smc_coherence_block(a, l) \ -{ \ - write_barrier(); \ - flush_pipeline(); \ -} +#define smc_coherence_block(a, l) \ + do { \ + write_barrier(); \ + flush_pipeline(); \ + } while (0) -#endif /* defined(US3) */ +#endif /* defined(US3) */ #endif Index: kernel/arch/sparc64/include/cycle.h =================================================================== --- kernel/arch/sparc64/include/cycle.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/cycle.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -36,7 +36,8 @@ #define KERN_sparc64_CYCLE_H_ -#include +#include +#include -static inline uint64_t get_cycle(void) +NO_TRACE static inline uint64_t get_cycle(void) { return tick_read(); Index: kernel/arch/sparc64/include/faddr.h =================================================================== --- kernel/arch/sparc64/include/faddr.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/faddr.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup sparc64 +/** @addtogroup sparc64 * @{ */ @@ -38,5 +38,5 @@ #include -#define FADDR(fptr) ((uintptr_t) (fptr)) +#define FADDR(fptr) ((uintptr_t) (fptr)) #endif Index: kernel/arch/sparc64/include/interrupt.h =================================================================== --- kernel/arch/sparc64/include/interrupt.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/interrupt.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -39,39 +39,43 @@ #include #include +#include -#define IVT_ITEMS 15 -#define IVT_FIRST 1 +#define IVT_ITEMS 15 +#define IVT_FIRST 1 /* This needs to be defined for inter-architecture API portability. */ -#define VECTOR_TLB_SHOOTDOWN_IPI 0 +#define VECTOR_TLB_SHOOTDOWN_IPI 0 enum { IPI_TLB_SHOOTDOWN = VECTOR_TLB_SHOOTDOWN_IPI -}; +}; typedef struct istate { - uint64_t tnpc; - uint64_t tpc; - uint64_t tstate; + uint64_t tnpc; + uint64_t tpc; + uint64_t tstate; } istate_t; -static inline void istate_set_retaddr(istate_t *istate, uintptr_t retaddr) +NO_TRACE static inline void istate_set_retaddr(istate_t *istate, + uintptr_t retaddr) { istate->tpc = retaddr; } -static inline int istate_from_uspace(istate_t *istate) +NO_TRACE static inline int istate_from_uspace(istate_t *istate) { return !(istate->tstate & TSTATE_PRIV_BIT); } -static inline unative_t istate_get_pc(istate_t *istate) +NO_TRACE static inline unative_t istate_get_pc(istate_t *istate) { return istate->tpc; } -static inline unative_t istate_get_fp(istate_t *istate) +NO_TRACE static inline unative_t istate_get_fp(istate_t *istate) { - return 0; /* TODO */ + /* TODO */ + + return 0; } Index: kernel/arch/sparc64/include/mm/as.h =================================================================== --- kernel/arch/sparc64/include/mm/as.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/mm/as.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup sparc64mm +/** @addtogroup sparc64mm * @{ */ @@ -37,7 +37,11 @@ #if defined (SUN4U) + #include + #elif defined (SUN4V) + #include + #endif Index: kernel/arch/sparc64/include/mm/frame.h =================================================================== --- kernel/arch/sparc64/include/mm/frame.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/mm/frame.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup sparc64mm +/** @addtogroup sparc64mm * @{ */ @@ -37,7 +37,11 @@ #if defined (SUN4U) + #include + #elif defined (SUN4V) + #include + #endif Index: kernel/arch/sparc64/include/mm/sun4u/tlb.h =================================================================== --- kernel/arch/sparc64/include/mm/sun4u/tlb.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/mm/sun4u/tlb.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -100,4 +100,5 @@ #include #include +#include #include #include @@ -242,5 +243,5 @@ * Determine the number of entries in the DMMU's small TLB. */ -static inline uint16_t tlb_dsmall_size(void) +NO_TRACE static inline uint16_t tlb_dsmall_size(void) { return 16; @@ -250,5 +251,5 @@ * Determine the number of entries in each DMMU's big TLB. */ -static inline uint16_t tlb_dbig_size(void) +NO_TRACE static inline uint16_t tlb_dbig_size(void) { return 512; @@ -258,5 +259,5 @@ * Determine the number of entries in the IMMU's small TLB. */ -static inline uint16_t tlb_ismall_size(void) +NO_TRACE static inline uint16_t tlb_ismall_size(void) { return 16; @@ -266,5 +267,5 @@ * Determine the number of entries in the IMMU's big TLB. */ -static inline uint16_t tlb_ibig_size(void) +NO_TRACE static inline uint16_t tlb_ibig_size(void) { if (((ver_reg_t) ver_read()).impl == IMPL_ULTRASPARCIV_PLUS) @@ -280,5 +281,5 @@ * @return Current value of Primary Context Register. */ -static inline uint64_t mmu_primary_context_read(void) +NO_TRACE static inline uint64_t mmu_primary_context_read(void) { return asi_u64_read(ASI_DMMU, VA_PRIMARY_CONTEXT_REG); @@ -289,5 +290,5 @@ * @param v New value of Primary Context Register. */ -static inline void mmu_primary_context_write(uint64_t v) +NO_TRACE static inline void mmu_primary_context_write(uint64_t v) { asi_u64_write(ASI_DMMU, VA_PRIMARY_CONTEXT_REG, v); @@ -299,5 +300,5 @@ * @return Current value of Secondary Context Register. */ -static inline uint64_t mmu_secondary_context_read(void) +NO_TRACE static inline uint64_t mmu_secondary_context_read(void) { return asi_u64_read(ASI_DMMU, VA_SECONDARY_CONTEXT_REG); @@ -308,5 +309,5 @@ * @param v New value of Primary Context Register. */ -static inline void mmu_secondary_context_write(uint64_t v) +NO_TRACE static inline void mmu_secondary_context_write(uint64_t v) { asi_u64_write(ASI_DMMU, VA_SECONDARY_CONTEXT_REG, v); @@ -323,5 +324,5 @@ * Register. */ -static inline uint64_t itlb_data_access_read(size_t entry) +NO_TRACE static inline uint64_t itlb_data_access_read(size_t entry) { itlb_data_access_addr_t reg; @@ -337,5 +338,5 @@ * @param value Value to be written. */ -static inline void itlb_data_access_write(size_t entry, uint64_t value) +NO_TRACE static inline void itlb_data_access_write(size_t entry, uint64_t value) { itlb_data_access_addr_t reg; @@ -354,5 +355,5 @@ * Register. */ -static inline uint64_t dtlb_data_access_read(size_t entry) +NO_TRACE static inline uint64_t dtlb_data_access_read(size_t entry) { dtlb_data_access_addr_t reg; @@ -368,5 +369,5 @@ * @param value Value to be written. */ -static inline void dtlb_data_access_write(size_t entry, uint64_t value) +NO_TRACE static inline void dtlb_data_access_write(size_t entry, uint64_t value) { dtlb_data_access_addr_t reg; @@ -384,5 +385,5 @@ * @return Current value of specified IMMU TLB Tag Read Register. */ -static inline uint64_t itlb_tag_read_read(size_t entry) +NO_TRACE static inline uint64_t itlb_tag_read_read(size_t entry) { itlb_tag_read_addr_t tag; @@ -399,5 +400,5 @@ * @return Current value of specified DMMU TLB Tag Read Register. */ -static inline uint64_t dtlb_tag_read_read(size_t entry) +NO_TRACE static inline uint64_t dtlb_tag_read_read(size_t entry) { dtlb_tag_read_addr_t tag; @@ -419,5 +420,5 @@ * Register. */ -static inline uint64_t itlb_data_access_read(int tlb, size_t entry) +NO_TRACE static inline uint64_t itlb_data_access_read(int tlb, size_t entry) { itlb_data_access_addr_t reg; @@ -434,5 +435,5 @@ * @param value Value to be written. */ -static inline void itlb_data_access_write(int tlb, size_t entry, +NO_TRACE static inline void itlb_data_access_write(int tlb, size_t entry, uint64_t value) { @@ -454,5 +455,5 @@ * Register. */ -static inline uint64_t dtlb_data_access_read(int tlb, size_t entry) +NO_TRACE static inline uint64_t dtlb_data_access_read(int tlb, size_t entry) { dtlb_data_access_addr_t reg; @@ -470,5 +471,5 @@ * @param value Value to be written. */ -static inline void dtlb_data_access_write(int tlb, size_t entry, +NO_TRACE static inline void dtlb_data_access_write(int tlb, size_t entry, uint64_t value) { @@ -489,5 +490,5 @@ * @return Current value of specified IMMU TLB Tag Read Register. */ -static inline uint64_t itlb_tag_read_read(int tlb, size_t entry) +NO_TRACE static inline uint64_t itlb_tag_read_read(int tlb, size_t entry) { itlb_tag_read_addr_t tag; @@ -506,5 +507,5 @@ * @return Current value of specified DMMU TLB Tag Read Register. */ -static inline uint64_t dtlb_tag_read_read(int tlb, size_t entry) +NO_TRACE static inline uint64_t dtlb_tag_read_read(int tlb, size_t entry) { dtlb_tag_read_addr_t tag; @@ -523,5 +524,5 @@ * @param v Value to be written. */ -static inline void itlb_tag_access_write(uint64_t v) +NO_TRACE static inline void itlb_tag_access_write(uint64_t v) { asi_u64_write(ASI_IMMU, VA_IMMU_TAG_ACCESS, v); @@ -533,5 +534,5 @@ * @return Current value of IMMU TLB Tag Access Register. */ -static inline uint64_t itlb_tag_access_read(void) +NO_TRACE static inline uint64_t itlb_tag_access_read(void) { return asi_u64_read(ASI_IMMU, VA_IMMU_TAG_ACCESS); @@ -542,5 +543,5 @@ * @param v Value to be written. */ -static inline void dtlb_tag_access_write(uint64_t v) +NO_TRACE static inline void dtlb_tag_access_write(uint64_t v) { asi_u64_write(ASI_DMMU, VA_DMMU_TAG_ACCESS, v); @@ -552,5 +553,5 @@ * @return Current value of DMMU TLB Tag Access Register. */ -static inline uint64_t dtlb_tag_access_read(void) +NO_TRACE static inline uint64_t dtlb_tag_access_read(void) { return asi_u64_read(ASI_DMMU, VA_DMMU_TAG_ACCESS); @@ -562,5 +563,5 @@ * @param v Value to be written. */ -static inline void itlb_data_in_write(uint64_t v) +NO_TRACE static inline void itlb_data_in_write(uint64_t v) { asi_u64_write(ASI_ITLB_DATA_IN_REG, 0, v); @@ -572,5 +573,5 @@ * @param v Value to be written. */ -static inline void dtlb_data_in_write(uint64_t v) +NO_TRACE static inline void dtlb_data_in_write(uint64_t v) { asi_u64_write(ASI_DTLB_DATA_IN_REG, 0, v); @@ -582,5 +583,5 @@ * @return Current content of I-SFSR register. */ -static inline uint64_t itlb_sfsr_read(void) +NO_TRACE static inline uint64_t itlb_sfsr_read(void) { return asi_u64_read(ASI_IMMU, VA_IMMU_SFSR); @@ -591,5 +592,5 @@ * @param v New value of I-SFSR register. */ -static inline void itlb_sfsr_write(uint64_t v) +NO_TRACE static inline void itlb_sfsr_write(uint64_t v) { asi_u64_write(ASI_IMMU, VA_IMMU_SFSR, v); @@ -601,5 +602,5 @@ * @return Current content of D-SFSR register. */ -static inline uint64_t dtlb_sfsr_read(void) +NO_TRACE static inline uint64_t dtlb_sfsr_read(void) { return asi_u64_read(ASI_DMMU, VA_DMMU_SFSR); @@ -610,5 +611,5 @@ * @param v New value of D-SFSR register. */ -static inline void dtlb_sfsr_write(uint64_t v) +NO_TRACE static inline void dtlb_sfsr_write(uint64_t v) { asi_u64_write(ASI_DMMU, VA_DMMU_SFSR, v); @@ -620,5 +621,5 @@ * @return Current content of D-SFAR register. */ -static inline uint64_t dtlb_sfar_read(void) +NO_TRACE static inline uint64_t dtlb_sfar_read(void) { return asi_u64_read(ASI_DMMU, VA_DMMU_SFAR); @@ -633,5 +634,5 @@ * @param page Address which is on the page to be demapped. */ -static inline void itlb_demap(int type, int context_encoding, uintptr_t page) +NO_TRACE static inline void itlb_demap(int type, int context_encoding, uintptr_t page) { tlb_demap_addr_t da; @@ -659,5 +660,5 @@ * @param page Address which is on the page to be demapped. */ -static inline void dtlb_demap(int type, int context_encoding, uintptr_t page) +NO_TRACE static inline void dtlb_demap(int type, int context_encoding, uintptr_t page) { tlb_demap_addr_t da; Index: kernel/arch/sparc64/include/mm/sun4v/tlb.h =================================================================== --- kernel/arch/sparc64/include/mm/sun4v/tlb.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/mm/sun4v/tlb.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -43,5 +43,5 @@ #include -#include +#include #include #include @@ -88,32 +88,32 @@ * @return Current value of Primary Context Register. */ -static inline uint64_t mmu_primary_context_read(void) +NO_TRACE static inline uint64_t mmu_primary_context_read(void) { return asi_u64_read(ASI_PRIMARY_CONTEXT_REG, VA_PRIMARY_CONTEXT_REG); } - + /** Write MMU Primary Context Register. * * @param v New value of Primary Context Register. */ -static inline void mmu_primary_context_write(uint64_t v) +NO_TRACE static inline void mmu_primary_context_write(uint64_t v) { asi_u64_write(ASI_PRIMARY_CONTEXT_REG, VA_PRIMARY_CONTEXT_REG, v); } - + /** Read MMU Secondary Context Register. * * @return Current value of Secondary Context Register. */ -static inline uint64_t mmu_secondary_context_read(void) +NO_TRACE static inline uint64_t mmu_secondary_context_read(void) { return asi_u64_read(ASI_SECONDARY_CONTEXT_REG, VA_SECONDARY_CONTEXT_REG); } - + /** Write MMU Secondary Context Register. * * @param v New value of Secondary Context Register. */ -static inline void mmu_secondary_context_write(uint64_t v) +NO_TRACE static inline void mmu_secondary_context_write(uint64_t v) { asi_u64_write(ASI_SECONDARY_CONTEXT_REG, VA_SECONDARY_CONTEXT_REG, v); @@ -126,5 +126,5 @@ * @param mmu_flag MMU_FLAG_DTLB, MMU_FLAG_ITLB or a combination of both */ -static inline void mmu_demap_ctx(int context, int mmu_flag) { +NO_TRACE static inline void mmu_demap_ctx(int context, int mmu_flag) { __hypercall_fast4(MMU_DEMAP_CTX, 0, 0, context, mmu_flag); } @@ -137,5 +137,5 @@ * @param mmu_flag MMU_FLAG_DTLB, MMU_FLAG_ITLB or a combination of both */ -static inline void mmu_demap_page(uintptr_t vaddr, int context, int mmu_flag) { +NO_TRACE static inline void mmu_demap_page(uintptr_t vaddr, int context, int mmu_flag) { __hypercall_fast5(MMU_DEMAP_PAGE, 0, 0, vaddr, context, mmu_flag); } Index: kernel/arch/sparc64/include/mm/tlb.h =================================================================== --- kernel/arch/sparc64/include/mm/tlb.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/mm/tlb.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup sparc64mm +/** @addtogroup sparc64mm * @{ */ @@ -36,9 +36,12 @@ #define KERN_sparc64_TLB_H_ +#if defined (SUN4U) -#if defined (SUN4U) #include + #elif defined (SUN4V) + #include + #endif Index: kernel/arch/sparc64/include/sun4u/asm.h =================================================================== --- kernel/arch/sparc64/include/sun4u/asm.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/sun4u/asm.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup sparc64 +/** @addtogroup sparc64 * @{ */ @@ -36,22 +36,27 @@ #define KERN_sparc64_sun4u_ASM_H_ -extern uint64_t read_from_ag_g7(void); -extern void write_to_ag_g6(uint64_t val); -extern void write_to_ag_g7(uint64_t val); -extern void write_to_ig_g6(uint64_t val); - +#include /** Read Version Register. * * @return Value of VER register. + * */ -static inline uint64_t ver_read(void) +NO_TRACE static inline uint64_t ver_read(void) { uint64_t v; - asm volatile ("rdpr %%ver, %0\n" : "=r" (v)); + asm volatile ( + "rdpr %%ver, %[v]\n" + : [v] "=r" (v) + ); return v; } + +extern uint64_t read_from_ag_g7(void); +extern void write_to_ag_g6(uint64_t); +extern void write_to_ag_g7(uint64_t); +extern void write_to_ig_g6(uint64_t); #endif Index: kernel/arch/sparc64/include/sun4u/cpu.h =================================================================== --- kernel/arch/sparc64/include/sun4u/cpu.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/sun4u/cpu.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -36,19 +36,19 @@ #define KERN_sparc64_sun4u_CPU_H_ -#define MANUF_FUJITSU 0x04 -#define MANUF_ULTRASPARC 0x17 /**< UltraSPARC I, UltraSPARC II */ -#define MANUF_SUN 0x3e +#define MANUF_FUJITSU 0x04 +#define MANUF_ULTRASPARC 0x17 /**< UltraSPARC I, UltraSPARC II */ +#define MANUF_SUN 0x3e -#define IMPL_ULTRASPARCI 0x10 -#define IMPL_ULTRASPARCII 0x11 -#define IMPL_ULTRASPARCII_I 0x12 -#define IMPL_ULTRASPARCII_E 0x13 -#define IMPL_ULTRASPARCIII 0x14 -#define IMPL_ULTRASPARCIII_PLUS 0x15 -#define IMPL_ULTRASPARCIII_I 0x16 -#define IMPL_ULTRASPARCIV 0x18 -#define IMPL_ULTRASPARCIV_PLUS 0x19 +#define IMPL_ULTRASPARCI 0x10 +#define IMPL_ULTRASPARCII 0x11 +#define IMPL_ULTRASPARCII_I 0x12 +#define IMPL_ULTRASPARCII_E 0x13 +#define IMPL_ULTRASPARCIII 0x14 +#define IMPL_ULTRASPARCIII_PLUS 0x15 +#define IMPL_ULTRASPARCIII_I 0x16 +#define IMPL_ULTRASPARCIV 0x18 +#define IMPL_ULTRASPARCIV_PLUS 0x19 -#define IMPL_SPARC64V 0x5 +#define IMPL_SPARC64V 0x5 #ifndef __ASM__ @@ -58,4 +58,5 @@ #include #include +#include #ifdef CONFIG_SMP @@ -64,21 +65,21 @@ typedef struct { - uint32_t mid; /**< Processor ID as read from - UPA_CONFIG/FIREPLANE_CONFIG. */ + uint32_t mid; /**< Processor ID as read from + UPA_CONFIG/FIREPLANE_CONFIG. */ ver_reg_t ver; - uint32_t clock_frequency; /**< Processor frequency in Hz. */ - uint64_t next_tick_cmpr; /**< Next clock interrupt should be - generated when the TICK register - matches this value. */ + uint32_t clock_frequency; /**< Processor frequency in Hz. */ + uint64_t next_tick_cmpr; /**< Next clock interrupt should be + generated when the TICK register + matches this value. */ } cpu_arch_t; - -/** - * Reads the module ID (agent ID/CPUID) of the current CPU. +/** Read the module ID (agent ID/CPUID) of the current CPU. + * */ -static inline uint32_t read_mid(void) +NO_TRACE static inline uint32_t read_mid(void) { uint64_t icbus_config = asi_u64_read(ASI_ICBUS_CONFIG, 0); icbus_config = icbus_config >> ICBUS_CONFIG_MID_SHIFT; + #if defined (US) return icbus_config & 0x1f; @@ -91,5 +92,5 @@ } -#endif +#endif #endif Index: kernel/arch/sparc64/include/sun4v/asm.h =================================================================== --- kernel/arch/sparc64/include/sun4v/asm.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/sun4v/asm.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -27,5 +27,5 @@ */ -/** @addtogroup sparc64 +/** @addtogroup sparc64 * @{ */ Index: kernel/arch/sparc64/include/sun4v/cpu.h =================================================================== --- kernel/arch/sparc64/include/sun4v/cpu.h (revision b5382d4f551b38012fe4db27dd79c095f5e09706) +++ kernel/arch/sparc64/include/sun4v/cpu.h (revision bbfdf62eecb75fb74d785cdd587affc613b7ffe9) @@ -37,8 +37,8 @@ /** Maximum number of virtual processors. */ -#define MAX_NUM_STRANDS 64 +#define MAX_NUM_STRANDS 64 /** Maximum number of logical processors in a processor core */ -#define MAX_CORE_STRANDS 8 +#define MAX_CORE_STRANDS 8 #ifndef __ASM__ @@ -59,17 +59,13 @@ typedef struct cpu_arch { - uint64_t id; /**< virtual processor ID */ - uint32_t clock_frequency; /**< Processor frequency in Hz. */ - uint64_t next_tick_cmpr; /**< Next clock interrupt should be - generated when the TICK register - matches this value. */ - exec_unit_t *exec_unit; /**< Physical core. */ - unsigned long proposed_nrdy; /**< Proposed No. of ready threads - so that cores are equally balanced. */ + uint64_t id; /**< virtual processor ID */ + uint32_t clock_frequency; /**< Processor frequency in Hz. */ + uint64_t next_tick_cmpr; /**< Next clock interrupt should be + generated when the TICK register + matches this value. */ + exec_unit_t *exec_unit; /**< Physical core. */ + unsigned long proposed_nrdy; /**< Proposed No. of ready threads + so that cores are equally balanced. */ } cpu_arch_t; - -#endif - -#ifdef __ASM__ #endif