Index: kernel/arch/amd64/include/arch/asm.h =================================================================== --- kernel/arch/amd64/include/arch/asm.h (revision 0f17bffb9edacd5345a8cb7bd518bccfb25c3e30) +++ kernel/arch/amd64/include/arch/asm.h (revision 811770c99b96760cdbd8d7f79f5e5bcb969a353e) @@ -208,49 +208,67 @@ } -/** Enable interrupts. - * - * Enable interrupts and return previous - * value of EFLAGS. - * - * @return Old interrupt priority level. - * - */ -NO_TRACE static inline ipl_t interrupts_enable(void) { - ipl_t v; - +NO_TRACE static inline uint64_t read_rflags(void) +{ + uint64_t rflags; + asm volatile ( "pushfq\n" "popq %[v]\n" - "sti\n" - : [v] "=r" (v) - ); - - return v; + : [v] "=r" (rflags) + ); + + return rflags; +} + +NO_TRACE static inline void write_rflags(uint64_t rflags) +{ + asm volatile ( + "pushq %[v]\n" + "popfq\n" + :: [v] "r" (rflags) + ); +} + +/** Return interrupt priority level. + * + * Return the current interrupt priority level. + * + * @return Current interrupt priority level. + */ +NO_TRACE static inline ipl_t interrupts_read(void) { + return (ipl_t) read_rflags(); +} + +/** Enable interrupts. + * + * Enable interrupts and return the previous interrupt priority level. + * + * @return Old interrupt priority level. + */ +NO_TRACE static inline ipl_t interrupts_enable(void) { + ipl_t ipl = interrupts_read(); + + asm volatile ("sti\n"); + + return ipl; } /** Disable interrupts. * - * Disable interrupts and return previous - * value of EFLAGS. + * Disable interrupts and return the previous interrupt priority level. * * @return Old interrupt priority level. - * */ NO_TRACE static inline ipl_t interrupts_disable(void) { - ipl_t v; - - asm volatile ( - "pushfq\n" - "popq %[v]\n" - "cli\n" - : [v] "=r" (v) - ); - - return v; + ipl_t ipl = interrupts_read(); + + asm volatile ("cli\n"); + + return ipl; } /** Restore interrupt priority level. * - * Restore EFLAGS. + * Restore the previously save interrupt priority level. * * @param ipl Saved interrupt priority level. @@ -258,28 +276,5 @@ */ NO_TRACE static inline void interrupts_restore(ipl_t ipl) { - asm volatile ( - "pushq %[ipl]\n" - "popfq\n" - :: [ipl] "r" (ipl) - ); -} - -/** Return interrupt priority level. - * - * Return EFLAFS. - * - * @return Current interrupt priority level. - * - */ -NO_TRACE static inline ipl_t interrupts_read(void) { - ipl_t v; - - asm volatile ( - "pushfq\n" - "popq %[v]\n" - : [v] "=r" (v) - ); - - return v; + write_rflags((uint64_t) ipl); } @@ -291,13 +286,5 @@ NO_TRACE static inline bool interrupts_disabled(void) { - ipl_t v; - - asm volatile ( - "pushfq\n" - "popq %[v]\n" - : [v] "=r" (v) - ); - - return ((v & RFLAGS_IF) == 0); + return ((read_rflags() & RFLAGS_IF) == 0); } @@ -324,21 +311,4 @@ return ((uint64_t) dx << 32) | ax; -} - -/** Enable local APIC - * - * Enable local APIC in MSR. - * - */ -NO_TRACE static inline void enable_l_apic_in_msr(void) -{ - asm volatile ( - "movl $0x1b, %%ecx\n" - "rdmsr\n" - "orl $(1 << 11),%%eax\n" - "orl $(0xfee00000),%%eax\n" - "wrmsr\n" - ::: "%eax", "%ecx", "%edx" - ); } @@ -426,7 +396,10 @@ GEN_READ_REG(cr0) +GEN_WRITE_REG(cr0) GEN_READ_REG(cr2) GEN_READ_REG(cr3) GEN_WRITE_REG(cr3) +GEN_READ_REG(cr4) +GEN_WRITE_REG(cr4) GEN_READ_REG(dr0) @@ -512,4 +485,6 @@ extern uintptr_t int_63; +extern void enable_l_apic_in_msr(void); + #endif Index: kernel/arch/amd64/include/arch/cpu.h =================================================================== --- kernel/arch/amd64/include/arch/cpu.h (revision 0f17bffb9edacd5345a8cb7bd518bccfb25c3e30) +++ kernel/arch/amd64/include/arch/cpu.h (revision 811770c99b96760cdbd8d7f79f5e5bcb969a353e) @@ -36,24 +36,39 @@ #define KERN_amd64_CPU_H_ -#define RFLAGS_CF (1 << 0) -#define RFLAGS_PF (1 << 2) -#define RFLAGS_AF (1 << 4) -#define RFLAGS_ZF (1 << 6) -#define RFLAGS_SF (1 << 7) -#define RFLAGS_TF (1 << 8) -#define RFLAGS_IF (1 << 9) -#define RFLAGS_DF (1 << 10) -#define RFLAGS_OF (1 << 11) -#define RFLAGS_NT (1 << 14) -#define RFLAGS_RF (1 << 16) +#define RFLAGS_CF (1 << 0) +#define RFLAGS_PF (1 << 2) +#define RFLAGS_AF (1 << 4) +#define RFLAGS_ZF (1 << 6) +#define RFLAGS_SF (1 << 7) +#define RFLAGS_TF (1 << 8) +#define RFLAGS_IF (1 << 9) +#define RFLAGS_DF (1 << 10) +#define RFLAGS_OF (1 << 11) +#define RFLAGS_IOPL (3 << 12) +#define RFLAGS_NT (1 << 14) +#define RFLAGS_RF (1 << 16) +#define RFLAGS_ID (1 << 21) -#define EFER_MSR_NUM 0xc0000080 -#define AMD_SCE_FLAG 0 -#define AMD_LME_FLAG 8 -#define AMD_LMA_FLAG 10 -#define AMD_FFXSR_FLAG 14 -#define AMD_NXE_FLAG 11 +#define CR0_MP (1 << 1) +#define CR0_EM (1 << 2) +#define CR0_TS (1 << 3) +#define CR0_AM (1 << 18) +#define CR0_PG (1 << 31) + +#define CR4_PAE (1 << 5) +#define CR4_OSFXSR (1 << 9) + +/* EFER bits */ +#define AMD_SCE (1 << 0) +#define AMD_LME (1 << 8) +#define AMD_LMA (1 << 10) +#define AMD_NXE (1 << 11) +#define AMD_FFXSR (1 << 14) + +#define AMD_APIC_BASE_GE (1 << 11) /* MSR registers */ +#define AMD_MSR_APIC_BASE 0x0000001b +#define AMD_MSR_EFER 0xc0000080 #define AMD_MSR_STAR 0xc0000081 #define AMD_MSR_LSTAR 0xc0000082 @@ -85,6 +100,4 @@ }; -extern void set_efer_flag(int flag); -extern uint64_t read_efer_flag(void); void cpu_setup_fpu(void); Index: kernel/arch/amd64/src/amd64.c =================================================================== --- kernel/arch/amd64/src/amd64.c (revision 0f17bffb9edacd5345a8cb7bd518bccfb25c3e30) +++ kernel/arch/amd64/src/amd64.c (revision 811770c99b96760cdbd8d7f79f5e5bcb969a353e) @@ -64,34 +64,4 @@ #endif -/** Disable I/O on non-privileged levels - * - * Clean IOPL(12,13) and NT(14) flags in EFLAGS register - */ -static void clean_IOPL_NT_flags(void) -{ - asm volatile ( - "pushfq\n" - "pop %%rax\n" - "and $~(0x7000), %%rax\n" - "pushq %%rax\n" - "popfq\n" - ::: "%rax" - ); -} - -/** Disable alignment check - * - * Clean AM(18) flag in CR0 register - */ -static void clean_AM_flag(void) -{ - asm volatile ( - "mov %%cr0, %%rax\n" - "and $~(0x40000), %%rax\n" - "mov %%rax, %%cr0\n" - ::: "%rax" - ); -} - /** Perform amd64-specific initialization before main_bsp() is called. * @@ -116,5 +86,5 @@ { /* Enable no-execute pages */ - set_efer_flag(AMD_NXE_FLAG); + write_msr(AMD_MSR_EFER, read_msr(AMD_MSR_EFER) | AMD_NXE); /* Enable FPU */ cpu_setup_fpu(); @@ -123,10 +93,8 @@ pm_init(); - /* Disable I/O on nonprivileged levels - * clear the NT (nested-thread) flag - */ - clean_IOPL_NT_flags(); + /* Disable I/O on nonprivileged levels, clear the nested-thread flag */ + write_rflags(read_rflags() & ~(RFLAGS_IOPL | RFLAGS_NT)); /* Disable alignment check */ - clean_AM_flag(); + write_cr0(read_cr0() & ~CR0_AM); if (config.cpu_active == 1) { Index: kernel/arch/amd64/src/asm.S =================================================================== --- kernel/arch/amd64/src/asm.S (revision 0f17bffb9edacd5345a8cb7bd518bccfb25c3e30) +++ kernel/arch/amd64/src/asm.S (revision 811770c99b96760cdbd8d7f79f5e5bcb969a353e) @@ -33,4 +33,5 @@ #include #include +#include .text @@ -93,5 +94,5 @@ /* Flip the ID bit */ - btcl $21, %edx + xorl $RFLAGS_ID, %edx /* Store RFLAGS */ @@ -102,6 +103,6 @@ /* Get the ID bit again */ popq %rdx - andl $(1 << 21), %eax - andl $(1 << 21), %edx + andl $RFLAGS_ID, %eax + andl $RFLAGS_ID, %edx /* 0 if not supported, 1 if supported */ @@ -127,17 +128,16 @@ FUNCTION_END(cpuid) -FUNCTION_BEGIN(set_efer_flag) - movl $0xc0000080, %ecx +/** Enable local APIC + * + * Enable local APIC in MSR. + * + */ +FUNCTION_BEGIN(enable_l_apic_in_msr) + movl $AMD_MSR_APIC_BASE, %ecx rdmsr - btsl %edi, %eax + orl $(L_APIC_BASE | AMD_APIC_BASE_GE), %eax wrmsr ret -FUNCTION_END(set_efer_flag) - -FUNCTION_BEGIN(read_efer_flag) - movl $0xc0000080, %ecx - rdmsr - ret -FUNCTION_END(read_efer_flag) +FUNCTION_END(enable_l_apic_in_msr) /* @@ -541,3 +541,2 @@ ret FUNCTION_END(early_putchar) - Index: kernel/arch/amd64/src/boot/multiboot.S =================================================================== --- kernel/arch/amd64/src/boot/multiboot.S (revision 0f17bffb9edacd5345a8cb7bd518bccfb25c3e30) +++ kernel/arch/amd64/src/boot/multiboot.S (revision 811770c99b96760cdbd8d7f79f5e5bcb969a353e) @@ -168,5 +168,5 @@ movl %cr4, %eax - btsl $5, %eax + orl $CR4_PAE, %eax movl %eax, %cr4 @@ -176,12 +176,12 @@ /* Enable long mode */ - movl $EFER_MSR_NUM, %ecx + movl $AMD_MSR_EFER, %ecx rdmsr /* read EFER */ - btsl $AMD_LME_FLAG, %eax /* set LME = 1 */ + orl $AMD_LME, %eax /* set LME = 1 */ wrmsr /* Enable paging to activate long mode (set CR0.PG = 1) */ movl %cr0, %eax - btsl $31, %eax + orl $CR0_PG, %eax movl %eax, %cr0 Index: kernel/arch/amd64/src/boot/multiboot2.S =================================================================== --- kernel/arch/amd64/src/boot/multiboot2.S (revision 0f17bffb9edacd5345a8cb7bd518bccfb25c3e30) +++ kernel/arch/amd64/src/boot/multiboot2.S (revision 811770c99b96760cdbd8d7f79f5e5bcb969a353e) @@ -209,5 +209,5 @@ movl %cr4, %eax - btsl $5, %eax + orl $CR4_PAE, %eax movl %eax, %cr4 @@ -217,12 +217,12 @@ /* Enable long mode */ - movl $EFER_MSR_NUM, %ecx + movl $AMD_MSR_EFER, %ecx rdmsr /* read EFER */ - btsl $AMD_LME_FLAG, %eax /* set LME = 1 */ + orl $AMD_LME, %eax /* set LME = 1 */ wrmsr /* Enable paging to activate long mode (set CR0.PG = 1) */ movl %cr0, %eax - btsl $31, %eax + orl $CR0_PG, %eax movl %eax, %cr0 Index: kernel/arch/amd64/src/cpu/cpu.c =================================================================== --- kernel/arch/amd64/src/cpu/cpu.c (revision 0f17bffb9edacd5345a8cb7bd518bccfb25c3e30) +++ kernel/arch/amd64/src/cpu/cpu.c (revision 811770c99b96760cdbd8d7f79f5e5bcb969a353e) @@ -76,15 +76,6 @@ void cpu_setup_fpu(void) { - asm volatile ( - "movq %%cr0, %%rax\n" - "btsq $1, %%rax\n" /* cr0.mp */ - "btrq $2, %%rax\n" /* cr0.em */ - "movq %%rax, %%cr0\n" - - "movq %%cr4, %%rax\n" - "bts $9, %%rax\n" /* cr4.osfxsr */ - "movq %%rax, %%cr4\n" - ::: "%rax" - ); + write_cr0((read_cr0() & ~CR0_EM) | CR0_MP); + write_cr4(read_cr4() | CR4_OSFXSR); } @@ -97,20 +88,10 @@ void fpu_disable(void) { - asm volatile ( - "mov %%cr0, %%rax\n" - "bts $3, %%rax\n" - "mov %%rax, %%cr0\n" - ::: "%rax" - ); + write_cr0(read_cr0() | CR0_TS); } void fpu_enable(void) { - asm volatile ( - "mov %%cr0, %%rax\n" - "btr $3, %%rax\n" - "mov %%rax, %%cr0\n" - ::: "%rax" - ); + write_cr0(read_cr0() & ~CR0_TS); } Index: kernel/arch/amd64/src/smp/ap.S =================================================================== --- kernel/arch/amd64/src/smp/ap.S (revision 0f17bffb9edacd5345a8cb7bd518bccfb25c3e30) +++ kernel/arch/amd64/src/smp/ap.S (revision 811770c99b96760cdbd8d7f79f5e5bcb969a353e) @@ -75,5 +75,5 @@ movl %cr4, %eax - btsl $5, %eax + orl $CR4_PAE, %eax movl %eax, %cr4 @@ -82,12 +82,12 @@ # Enable long mode - movl $EFER_MSR_NUM, %ecx # EFER MSR number + movl $AMD_MSR_EFER, %ecx # EFER MSR number rdmsr # Read EFER - btsl $AMD_LME_FLAG, %eax # Set LME=1 + orl $AMD_LME, %eax # Set LME=1 wrmsr # Write EFER # Enable paging to activate long mode (set CR0.PG = 1) movl %cr0, %eax - btsl $31, %eax + orl $CR0_PG, %eax movl %eax, %cr0 Index: kernel/arch/amd64/src/syscall.c =================================================================== --- kernel/arch/amd64/src/syscall.c (revision 0f17bffb9edacd5345a8cb7bd518bccfb25c3e30) +++ kernel/arch/amd64/src/syscall.c (revision 811770c99b96760cdbd8d7f79f5e5bcb969a353e) @@ -48,5 +48,5 @@ { /* Enable SYSCALL/SYSRET */ - set_efer_flag(AMD_SCE_FLAG); + write_msr(AMD_MSR_EFER, read_msr(AMD_MSR_EFER) | AMD_SCE); /* Setup syscall entry address */ Index: kernel/arch/amd64/src/userspace.c =================================================================== --- kernel/arch/amd64/src/userspace.c (revision 0f17bffb9edacd5345a8cb7bd518bccfb25c3e30) +++ kernel/arch/amd64/src/userspace.c (revision 811770c99b96760cdbd8d7f79f5e5bcb969a353e) @@ -48,13 +48,13 @@ void userspace(uspace_arg_t *kernel_uarg) { - ipl_t ipl = interrupts_disable(); + uint64_t rflags = read_rflags(); - ipl &= ~(RFLAGS_CF | RFLAGS_PF | RFLAGS_AF | RFLAGS_ZF | RFLAGS_SF | - RFLAGS_DF | RFLAGS_OF); + rflags &= ~RFLAGS_NT; + rflags |= RFLAGS_IF; asm volatile ( "pushq %[udata_des]\n" "pushq %[stack_top]\n" - "pushq %[ipl]\n" + "pushq %[rflags]\n" "pushq %[utext_des]\n" "pushq %[entry]\n" @@ -67,5 +67,5 @@ [stack_top] "r" ((uint8_t *) kernel_uarg->uspace_stack + kernel_uarg->uspace_stack_size), - [ipl] "r" (ipl), + [rflags] "r" (rflags), [utext_des] "i" (GDT_SELECTOR(UTEXT_DES) | PL_USER), [entry] "r" (kernel_uarg->uspace_entry),