Index: kernel/arch/amd64/Makefile.inc =================================================================== --- kernel/arch/amd64/Makefile.inc (revision 33dac7d46d9290e712c0f579d477a3c7c97536a5) +++ kernel/arch/amd64/Makefile.inc (revision a1f60f34d64053b65db351e780b9315fdd0384a3) @@ -33,5 +33,5 @@ FPU_NO_CFLAGS = -mno-sse -mno-sse2 -CMN1 = -m64 -mcmodel=kernel -mno-red-zone -fno-unwind-tables -fno-omit-frame-pointer +CMN1 = -m64 -mcmodel=large -mno-red-zone -fno-unwind-tables -fno-omit-frame-pointer GCC_CFLAGS += $(CMN1) ICC_CFLAGS += $(CMN1) Index: kernel/arch/amd64/include/mm/as.h =================================================================== --- kernel/arch/amd64/include/mm/as.h (revision 33dac7d46d9290e712c0f579d477a3c7c97536a5) +++ kernel/arch/amd64/include/mm/as.h (revision a1f60f34d64053b65db351e780b9315fdd0384a3) @@ -27,5 +27,5 @@ */ -/** @addtogroup amd64mm +/** @addtogroup amd64mm * @{ */ @@ -36,16 +36,18 @@ #define KERN_amd64_AS_H_ -#define KERNEL_ADDRESS_SPACE_SHADOWED_ARCH 0 +#define KERNEL_ADDRESS_SPACE_SHADOWED_ARCH 0 -#define KERNEL_ADDRESS_SPACE_START_ARCH (unsigned long) 0xffff800000000000 -#define KERNEL_ADDRESS_SPACE_END_ARCH (unsigned long) 0xffffffff80000000 -#define USER_ADDRESS_SPACE_START_ARCH (unsigned long) 0x0000000000000000 -#define USER_ADDRESS_SPACE_END_ARCH (unsigned long) 0x00007fffffffffff +#define KERNEL_ADDRESS_SPACE_START_ARCH (unsigned long) 0xffff800000000000 +#define KERNEL_ADDRESS_SPACE_END_ARCH (unsigned long) 0xffffffffffffffff -#define USTACK_ADDRESS_ARCH (USER_ADDRESS_SPACE_END_ARCH-(PAGE_SIZE-1)) +#define USER_ADDRESS_SPACE_START_ARCH (unsigned long) 0x0000000000000000 +#define USER_ADDRESS_SPACE_END_ARCH (unsigned long) 0x00007fffffffffff -#define as_constructor_arch(as, flags) (as != as) -#define as_destructor_arch(as) (as != as) -#define as_create_arch(as, flags) (as != as) +#define USTACK_ADDRESS_ARCH (USER_ADDRESS_SPACE_END_ARCH - (PAGE_SIZE - 1)) + +#define as_constructor_arch(as, flags) (as != as) +#define as_destructor_arch(as) (as != as) +#define as_create_arch(as, flags) (as != as) + #define as_install_arch(as) #define as_deinstall_arch(as) Index: kernel/arch/amd64/include/mm/page.h =================================================================== --- kernel/arch/amd64/include/mm/page.h (revision 33dac7d46d9290e712c0f579d477a3c7c97536a5) +++ kernel/arch/amd64/include/mm/page.h (revision a1f60f34d64053b65db351e780b9315fdd0384a3) @@ -35,12 +35,9 @@ /** Paging on AMD64 * - * The space is divided in positive numbers - userspace and - * negative numbers - kernel space. The 'negative' space starting - * with 0xffff800000000000 and ending with 0xffffffff80000000 - * (-2GB) is identically mapped physical memory. The area - * (0xffffffff80000000 ... 0xffffffffffffffff is again identically - * mapped first 2GB. - * - * ATTENTION - PA2KA(KA2PA(x)) != x if 'x' is in kernel + * The space is divided in positive numbers (uspace) and + * negative numbers (kernel). The 'negative' space starting + * with 0xffff800000000000 and ending with 0xffffffffffffffff + * is identically mapped physical memory. + * */ @@ -50,47 +47,38 @@ #include -#define PAGE_WIDTH FRAME_WIDTH -#define PAGE_SIZE FRAME_SIZE +#define PAGE_WIDTH FRAME_WIDTH +#define PAGE_SIZE FRAME_SIZE #ifdef KERNEL #ifndef __ASM__ -# include -# include -# include - -static inline uintptr_t ka2pa(uintptr_t x) -{ - if (x > 0xffffffff80000000) - return x - 0xffffffff80000000; - else - return x - 0xffff800000000000; -} - -# define KA2PA(x) ka2pa((uintptr_t) x) -# define PA2KA_CODE(x) (((uintptr_t) (x)) + 0xffffffff80000000) -# define PA2KA(x) (((uintptr_t) (x)) + 0xffff800000000000) -#else -# define KA2PA(x) ((x) - 0xffffffff80000000) -# define PA2KA(x) ((x) + 0xffffffff80000000) -#endif + +#define KA2PA(x) (((uintptr_t) (x)) - 0xffff800000000000) +#define PA2KA(x) (((uintptr_t) (x)) + 0xffff800000000000) + +#else /* __ASM__ */ + +#define KA2PA(x) ((x) - 0xffff800000000000) +#define PA2KA(x) ((x) + 0xffff800000000000) + +#endif /* __ASM__ */ /* Number of entries in each level. */ -#define PTL0_ENTRIES_ARCH 512 -#define PTL1_ENTRIES_ARCH 512 -#define PTL2_ENTRIES_ARCH 512 -#define PTL3_ENTRIES_ARCH 512 +#define PTL0_ENTRIES_ARCH 512 +#define PTL1_ENTRIES_ARCH 512 +#define PTL2_ENTRIES_ARCH 512 +#define PTL3_ENTRIES_ARCH 512 /* Page table sizes for each level. */ -#define PTL0_SIZE_ARCH ONE_FRAME -#define PTL1_SIZE_ARCH ONE_FRAME -#define PTL2_SIZE_ARCH ONE_FRAME -#define PTL3_SIZE_ARCH ONE_FRAME +#define PTL0_SIZE_ARCH ONE_FRAME +#define PTL1_SIZE_ARCH ONE_FRAME +#define PTL2_SIZE_ARCH ONE_FRAME +#define PTL3_SIZE_ARCH ONE_FRAME /* Macros calculating indices into page tables in each level. */ -#define PTL0_INDEX_ARCH(vaddr) (((vaddr) >> 39) & 0x1ff) -#define PTL1_INDEX_ARCH(vaddr) (((vaddr) >> 30) & 0x1ff) -#define PTL2_INDEX_ARCH(vaddr) (((vaddr) >> 21) & 0x1ff) -#define PTL3_INDEX_ARCH(vaddr) (((vaddr) >> 12) & 0x1ff) +#define PTL0_INDEX_ARCH(vaddr) (((vaddr) >> 39) & 0x1ff) +#define PTL1_INDEX_ARCH(vaddr) (((vaddr) >> 30) & 0x1ff) +#define PTL2_INDEX_ARCH(vaddr) (((vaddr) >> 21) & 0x1ff) +#define PTL3_INDEX_ARCH(vaddr) (((vaddr) >> 12) & 0x1ff) /* Get PTE address accessors for each level. */ @@ -156,4 +144,8 @@ #ifndef __ASM__ +#include +#include +#include + /* Page fault error codes. */ @@ -161,36 +153,36 @@ * page. */ -#define PFERR_CODE_P (1 << 0) +#define PFERR_CODE_P (1 << 0) /** When bit on this position is 1, the page fault was caused by a write. */ -#define PFERR_CODE_RW (1 << 1) +#define PFERR_CODE_RW (1 << 1) /** When bit on this position is 1, the page fault was caused in user mode. */ -#define PFERR_CODE_US (1 << 2) +#define PFERR_CODE_US (1 << 2) /** When bit on this position is 1, a reserved bit was set in page directory. */ -#define PFERR_CODE_RSVD (1 << 3) +#define PFERR_CODE_RSVD (1 << 3) /** When bit on this position os 1, the page fault was caused during instruction * fecth. */ -#define PFERR_CODE_ID (1 << 4) +#define PFERR_CODE_ID (1 << 4) /** Page Table Entry. */ typedef struct { - unsigned present : 1; - unsigned writeable : 1; - unsigned uaccessible : 1; - unsigned page_write_through : 1; - unsigned page_cache_disable : 1; - unsigned accessed : 1; - unsigned dirty : 1; - unsigned unused: 1; - unsigned global : 1; - unsigned soft_valid : 1; /**< Valid content even if present bit is cleared. */ - unsigned avl : 2; - unsigned addr_12_31 : 30; - unsigned addr_32_51 : 21; - unsigned no_execute : 1; + unsigned int present : 1; + unsigned int writeable : 1; + unsigned int uaccessible : 1; + unsigned int page_write_through : 1; + unsigned int page_cache_disable : 1; + unsigned int accessed : 1; + unsigned int dirty : 1; + unsigned int unused: 1; + unsigned int global : 1; + unsigned int soft_valid : 1; /**< Valid content even if present bit is cleared. */ + unsigned int avl : 2; + unsigned int addr_12_31 : 30; + unsigned int addr_32_51 : 21; + unsigned int no_execute : 1; } __attribute__ ((packed)) pte_t; @@ -211,5 +203,5 @@ { pte_t *p = &pt[i]; - + p->addr_12_31 = (a >> 12) & 0xfffff; p->addr_32_51 = a >> 32; Index: kernel/arch/amd64/src/asm_utils.S =================================================================== --- kernel/arch/amd64/src/asm_utils.S (revision 33dac7d46d9290e712c0f579d477a3c7c97536a5) +++ kernel/arch/amd64/src/asm_utils.S (revision a1f60f34d64053b65db351e780b9315fdd0384a3) @@ -27,24 +27,25 @@ # -#define IREGISTER_SPACE 80 - -#define IOFFSET_RAX 0x0 -#define IOFFSET_RCX 0x8 -#define IOFFSET_RDX 0x10 -#define IOFFSET_RSI 0x18 -#define IOFFSET_RDI 0x20 -#define IOFFSET_R8 0x28 -#define IOFFSET_R9 0x30 -#define IOFFSET_R10 0x38 -#define IOFFSET_R11 0x40 -#define IOFFSET_RBP 0x48 - -# Mask for interrupts 0 - 31 (bits 0 - 31) where 0 means that int has no error word -# and 1 means interrupt with error word -#define ERROR_WORD_INTERRUPT_LIST 0x00027D00 +#define IREGISTER_SPACE 80 + +#define IOFFSET_RAX 0x00 +#define IOFFSET_RCX 0x08 +#define IOFFSET_RDX 0x10 +#define IOFFSET_RSI 0x18 +#define IOFFSET_RDI 0x20 +#define IOFFSET_R8 0x28 +#define IOFFSET_R9 0x30 +#define IOFFSET_R10 0x38 +#define IOFFSET_R11 0x40 +#define IOFFSET_RBP 0x48 + +# Mask for interrupts 0 - 31 (bits 0 - 31) where 0 means that int +# has no error word and 1 means interrupt with error word + +#define ERROR_WORD_INTERRUPT_LIST 0x00027D00 #include #include - + .text .global interrupt_handlers @@ -53,5 +54,6 @@ panic_printf: - movq $halt, (%rsp) + movabsq $halt, %rax + movq %rax, (%rsp) jmp printf @@ -76,7 +78,7 @@ jmp _memsetw -#define MEMCPY_DST %rdi -#define MEMCPY_SRC %rsi -#define MEMCPY_SIZE %rdx +#define MEMCPY_DST %rdi +#define MEMCPY_SRC %rsi +#define MEMCPY_SIZE %rdx /** @@ -89,9 +91,10 @@ * or copy_to_uspace(). * - * @param MEMCPY_DST Destination address. - * @param MEMCPY_SRC Source address. - * @param MEMCPY_SIZE Number of bytes to copy. + * @param MEMCPY_DST Destination address. + * @param MEMCPY_SRC Source address. + * @param MEMCPY_SIZE Number of bytes to copy. * * @retrun MEMCPY_DST on success, 0 on failure. + * */ memcpy: @@ -99,22 +102,22 @@ memcpy_to_uspace: movq MEMCPY_DST, %rax - + movq MEMCPY_SIZE, %rcx - shrq $3, %rcx /* size / 8 */ - - rep movsq /* copy as much as possible word by word */ - + shrq $3, %rcx /* size / 8 */ + + rep movsq /* copy as much as possible word by word */ + movq MEMCPY_SIZE, %rcx - andq $7, %rcx /* size % 8 */ + andq $7, %rcx /* size % 8 */ jz 0f - rep movsb /* copy the rest byte by byte */ - -0: - ret /* return MEMCPY_SRC, success */ + rep movsb /* copy the rest byte by byte */ + + 0: + ret /* return MEMCPY_SRC, success */ memcpy_from_uspace_failover_address: memcpy_to_uspace_failover_address: - xorq %rax, %rax /* return 0, failure */ + xorq %rax, %rax /* return 0, failure */ ret @@ -124,28 +127,28 @@ # has_cpuid: - pushfq # store flags - popq %rax # read flags - movq %rax,%rdx # copy flags - btcl $21,%edx # swap the ID bit + pushfq # store flags + popq %rax # read flags + movq %rax, %rdx # copy flags + btcl $21, %edx # swap the ID bit pushq %rdx - popfq # propagate the change into flags + popfq # propagate the change into flags pushfq - popq %rdx # read flags - andl $(1<<21),%eax # interested only in ID bit - andl $(1<<21),%edx - xorl %edx,%eax # 0 if not supported, 1 if supported + popq %rdx # read flags + andl $(1 << 21), %eax # interested only in ID bit + andl $(1 << 21), %edx + xorl %edx, %eax # 0 if not supported, 1 if supported ret cpuid: - movq %rbx, %r10 # we have to preserve rbx across function calls - - movl %edi,%eax # load the command into %eax - - cpuid - movl %eax,0(%rsi) - movl %ebx,4(%rsi) - movl %ecx,8(%rsi) - movl %edx,12(%rsi) - + movq %rbx, %r10 # we have to preserve rbx across function calls + + movl %edi,%eax # load the command into %eax + + cpuid + movl %eax, 0(%rsi) + movl %ebx, 4(%rsi) + movl %ecx, 8(%rsi) + movl %edx, 12(%rsi) + movq %r10, %rbx ret @@ -157,9 +160,9 @@ wrmsr ret - + read_efer_flag: movq $0xc0000080, %rcx rdmsr - ret + ret # Push all volatile general purpose registers on stack @@ -190,6 +193,6 @@ .endm -#define INTERRUPT_ALIGN 128 - +#define INTERRUPT_ALIGN 128 + ## Declare interrupt handlers # @@ -200,5 +203,5 @@ # .macro handler i n - + /* * Choose between version with error code and version without error @@ -209,5 +212,5 @@ * Therefore we align the interrupt handlers. */ - + .iflt \i-32 .if (1 << \i) & ERROR_WORD_INTERRUPT_LIST @@ -220,5 +223,5 @@ * Version without error word, */ - subq $(IREGISTER_SPACE+8), %rsp + subq $(IREGISTER_SPACE + 8), %rsp .endif .else @@ -226,25 +229,25 @@ * Version without error word, */ - subq $(IREGISTER_SPACE+8), %rsp - .endif - + subq $(IREGISTER_SPACE + 8), %rsp + .endif + save_all_gpr cld - + # Stop stack traces here xorq %rbp, %rbp - - movq $(\i), %rdi # %rdi - first parameter - movq %rsp, %rsi # %rsi - pointer to istate - call exc_dispatch # exc_dispatch(i, istate) + + movq $(\i), %rdi # %rdi - first parameter + movq %rsp, %rsi # %rsi - pointer to istate + call exc_dispatch # exc_dispatch(i, istate) restore_all_gpr # $8 = Skip error word - addq $(IREGISTER_SPACE+8), %rsp + addq $(IREGISTER_SPACE + 8), %rsp iretq - + .align INTERRUPT_ALIGN - .if (\n-\i)-1 - handler "(\i+1)",\n + .if (\n - \i) - 1 + handler "(\i + 1)", \n .endif .endm @@ -252,53 +255,53 @@ .align INTERRUPT_ALIGN interrupt_handlers: -h_start: - handler 0 IDT_ITEMS -h_end: + h_start: + handler 0 IDT_ITEMS + h_end: ## Low-level syscall handler -# +# # Registers on entry: # -# @param rcx Userspace return address. -# @param r11 Userspace RLFAGS. -# -# @param rax Syscall number. -# @param rdi 1st syscall argument. -# @param rsi 2nd syscall argument. -# @param rdx 3rd syscall argument. -# @param r10 4th syscall argument. Used instead of RCX because the -# SYSCALL instruction clobbers it. -# @param r8 5th syscall argument. -# @param r9 6th syscall argument. -# -# @return Return value is in rax. +# @param rcx Userspace return address. +# @param r11 Userspace RLFAGS. +# +# @param rax Syscall number. +# @param rdi 1st syscall argument. +# @param rsi 2nd syscall argument. +# @param rdx 3rd syscall argument. +# @param r10 4th syscall argument. Used instead of RCX because +# the SYSCALL instruction clobbers it. +# @param r8 5th syscall argument. +# @param r9 6th syscall argument. +# +# @return Return value is in rax. # syscall_entry: - swapgs # Switch to hidden gs - # - # %gs:0 Scratch space for this thread's user RSP - # %gs:8 Address to be used as this thread's kernel RSP + swapgs # Switch to hidden gs # - movq %rsp, %gs:0 # Save this thread's user RSP - movq %gs:8, %rsp # Set this thread's kernel RSP - swapgs # Switch back to remain consistent + # %gs:0 Scratch space for this thread's user RSP + # %gs:8 Address to be used as this thread's kernel RSP + # + movq %rsp, %gs:0 # Save this thread's user RSP + movq %gs:8, %rsp # Set this thread's kernel RSP + swapgs # Switch back to remain consistent sti pushq %rcx pushq %r11 - - movq %r10, %rcx # Copy the 4th argument where it is expected + + movq %r10, %rcx # Copy the 4th argument where it is expected pushq %rax call syscall_handler addq $8, %rsp - + popq %r11 popq %rcx - + cli swapgs - movq %gs:0, %rsp # Restore the user RSP + movq %gs:0, %rsp # Restore the user RSP swapgs - + sysretq @@ -306,3 +309,3 @@ .global interrupt_handler_size -interrupt_handler_size: .quad (h_end-h_start)/IDT_ITEMS +interrupt_handler_size: .quad (h_end - h_start) / IDT_ITEMS Index: kernel/arch/amd64/src/boot/boot.S =================================================================== --- kernel/arch/amd64/src/boot/boot.S (revision 33dac7d46d9290e712c0f579d477a3c7c97536a5) +++ kernel/arch/amd64/src/boot/boot.S (revision a1f60f34d64053b65db351e780b9315fdd0384a3) @@ -31,5 +31,5 @@ #include #include -#include +#include #include #include @@ -172,11 +172,14 @@ xorq %rsi, %rsi movl grub_ebx, %esi - call arch_pre_main + + movabsq $arch_pre_main, %rax + callq *%rax # create the first stack frame pushq $0 movq %rsp, %rbp - - call main_bsp + + movabsq $main_bsp, %rax + call *%rax # not reached @@ -256,10 +259,10 @@ # # Macro for generating initial page table contents. -# @param cnt Number of entries to generat. Must be multiple of 8. +# @param cnt Number of entries to generate. Must be multiple of 8. # @param g Number of GB that will be added to the mapping. # -.macro ptl2gen cnt g +.macro ptl2gen cnt g .if \cnt - ptl2gen "\cnt - 8" \g + ptl2gen "\cnt - 8" \g .quad ((\cnt - 8) * 0x200000) + (\g * 1024 * 1024 * 1024) | (PTL_WRITABLE | PTL_PRESENT | PTL_2MB_PAGE) .quad ((\cnt - 7) * 0x200000) + (\g * 1024 * 1024 * 1024) | (PTL_WRITABLE | PTL_PRESENT | PTL_2MB_PAGE) @@ -276,5 +279,5 @@ .align 4096 .global ptl_2_0g -ptl_2_0g: +ptl_2_0g: ptl2gen 512 0 @@ -302,11 +305,8 @@ # Identity mapping for [0; 4G) .quad ptl_2_0g + (PTL_WRITABLE | PTL_PRESENT) - .quad ptl_2_1g + (PTL_WRITABLE | PTL_PRESENT) + .quad ptl_2_1g + (PTL_WRITABLE | PTL_PRESENT) .quad ptl_2_2g + (PTL_WRITABLE | PTL_PRESENT) .quad ptl_2_3g + (PTL_WRITABLE | PTL_PRESENT) - .fill 506, 8, 0 - # Mapping of [0; 1G) at -2G - .quad ptl_2_0g + (PTL_WRITABLE | PTL_PRESENT) - .fill 1, 8, 0 + .fill 508, 8, 0 .align 4096 @@ -314,8 +314,7 @@ ptl_0: .quad ptl_1 + (PTL_WRITABLE | PTL_PRESENT) - .fill 255,8,0 + .fill 255, 8, 0 .quad ptl_1 + (PTL_WRITABLE | PTL_PRESENT) - .fill 254,8,0 - .quad ptl_1 + (PTL_WRITABLE | PTL_PRESENT) + .fill 255, 8, 0 .section K_DATA_START, "aw", @progbits Index: kernel/arch/amd64/src/context.S =================================================================== --- kernel/arch/amd64/src/context.S (revision 33dac7d46d9290e712c0f579d477a3c7c97536a5) +++ kernel/arch/amd64/src/context.S (revision a1f60f34d64053b65db351e780b9315fdd0384a3) @@ -41,9 +41,9 @@ context_save_arch: movq (%rsp), %rdx # the caller's return %eip - - # In %edi is passed 1st argument - CONTEXT_SAVE_ARCH_CORE %rdi %rdx - xorq %rax,%rax # context_save returns 1 + # 1st argument passed in %edi + CONTEXT_SAVE_ARCH_CORE %rdi %rdx + + xorq %rax, %rax # context_save returns 1 incq %rax ret @@ -55,10 +55,9 @@ # pointed by the 1st argument. Returns 0 in EAX. # -context_restore_arch: - +context_restore_arch: CONTEXT_RESTORE_ARCH_CORE %rdi %rdx - - movq %rdx,(%rsp) - - xorq %rax,%rax # context_restore returns 0 + + movq %rdx, (%rsp) + + xorq %rax, %rax # context_restore returns 0 ret Index: kernel/arch/amd64/src/fpu_context.c =================================================================== --- kernel/arch/amd64/src/fpu_context.c (revision 33dac7d46d9290e712c0f579d477a3c7c97536a5) +++ kernel/arch/amd64/src/fpu_context.c (revision a1f60f34d64053b65db351e780b9315fdd0384a3) @@ -27,5 +27,5 @@ */ -/** @addtogroup amd64 +/** @addtogroup amd64 * @{ */ Index: kernel/arch/amd64/src/mm/page.c =================================================================== --- kernel/arch/amd64/src/mm/page.c (revision 33dac7d46d9290e712c0f579d477a3c7c97536a5) +++ kernel/arch/amd64/src/mm/page.c (revision a1f60f34d64053b65db351e780b9315fdd0384a3) @@ -52,68 +52,70 @@ pte_t helper_ptl2[512] __attribute__((aligned (PAGE_SIZE))); pte_t helper_ptl3[512] __attribute__((aligned (PAGE_SIZE))); -extern pte_t ptl_0; /* From boot.S */ - -#define PTL1_PRESENT(ptl0, page) (!(GET_PTL1_FLAGS_ARCH(ptl0, PTL0_INDEX_ARCH(page)) & PAGE_NOT_PRESENT)) -#define PTL2_PRESENT(ptl1, page) (!(GET_PTL2_FLAGS_ARCH(ptl1, PTL1_INDEX_ARCH(page)) & PAGE_NOT_PRESENT)) -#define PTL3_PRESENT(ptl2, page) (!(GET_PTL3_FLAGS_ARCH(ptl2, PTL2_INDEX_ARCH(page)) & PAGE_NOT_PRESENT)) - -#define PTL1_ADDR(ptl0, page) ((pte_t *)PA2KA(GET_PTL1_ADDRESS_ARCH(ptl0, PTL0_INDEX_ARCH(page)))) -#define PTL2_ADDR(ptl1, page) ((pte_t *)PA2KA(GET_PTL2_ADDRESS_ARCH(ptl1, PTL1_INDEX_ARCH(page)))) -#define PTL3_ADDR(ptl2, page) ((pte_t *)PA2KA(GET_PTL3_ADDRESS_ARCH(ptl2, PTL2_INDEX_ARCH(page)))) - -#define SETUP_PTL1(ptl0, page, tgt) { \ - SET_PTL1_ADDRESS_ARCH(ptl0, PTL0_INDEX_ARCH(page), (uintptr_t)KA2PA(tgt)); \ - SET_PTL1_FLAGS_ARCH(ptl0, PTL0_INDEX_ARCH(page), PAGE_WRITE | PAGE_EXEC); \ - } -#define SETUP_PTL2(ptl1, page, tgt) { \ - SET_PTL2_ADDRESS_ARCH(ptl1, PTL1_INDEX_ARCH(page), (uintptr_t)KA2PA(tgt)); \ - SET_PTL2_FLAGS_ARCH(ptl1, PTL1_INDEX_ARCH(page), PAGE_WRITE | PAGE_EXEC); \ - } -#define SETUP_PTL3(ptl2, page, tgt) { \ - SET_PTL3_ADDRESS_ARCH(ptl2, PTL2_INDEX_ARCH(page), (uintptr_t)KA2PA(tgt)); \ - SET_PTL3_FLAGS_ARCH(ptl2, PTL2_INDEX_ARCH(page), PAGE_WRITE | PAGE_EXEC); \ - } -#define SETUP_FRAME(ptl3, page, tgt) { \ - SET_FRAME_ADDRESS_ARCH(ptl3, PTL3_INDEX_ARCH(page), (uintptr_t)KA2PA(tgt)); \ - SET_FRAME_FLAGS_ARCH(ptl3, PTL3_INDEX_ARCH(page), PAGE_WRITE | PAGE_EXEC); \ - } - + +static uintptr_t oldpage = 0; + +extern pte_t ptl_0; /* From boot.S */ + +#define PTL1_PRESENT(ptl0, page) \ + (!(GET_PTL1_FLAGS_ARCH(ptl0, PTL0_INDEX_ARCH(page)) & PAGE_NOT_PRESENT)) + +#define PTL2_PRESENT(ptl1, page) \ + (!(GET_PTL2_FLAGS_ARCH(ptl1, PTL1_INDEX_ARCH(page)) & PAGE_NOT_PRESENT)) + +#define PTL3_PRESENT(ptl2, page) \ + (!(GET_PTL3_FLAGS_ARCH(ptl2, PTL2_INDEX_ARCH(page)) & PAGE_NOT_PRESENT)) + +#define PTL1_ADDR(ptl0, page) \ + ((pte_t *) PA2KA(GET_PTL1_ADDRESS_ARCH(ptl0, PTL0_INDEX_ARCH(page)))) + +#define PTL2_ADDR(ptl1, page) \ + ((pte_t *) PA2KA(GET_PTL2_ADDRESS_ARCH(ptl1, PTL1_INDEX_ARCH(page)))) + +#define PTL3_ADDR(ptl2, page) \ + ((pte_t *) PA2KA(GET_PTL3_ADDRESS_ARCH(ptl2, PTL2_INDEX_ARCH(page)))) + +#define SETUP_PTL1(ptl0, page, tgt) \ + { \ + SET_PTL1_ADDRESS_ARCH(ptl0, PTL0_INDEX_ARCH(page), (uintptr_t)KA2PA(tgt)); \ + SET_PTL1_FLAGS_ARCH(ptl0, PTL0_INDEX_ARCH(page), PAGE_WRITE | PAGE_EXEC); \ + } + +#define SETUP_PTL2(ptl1, page, tgt) \ + { \ + SET_PTL2_ADDRESS_ARCH(ptl1, PTL1_INDEX_ARCH(page), (uintptr_t)KA2PA(tgt)); \ + SET_PTL2_FLAGS_ARCH(ptl1, PTL1_INDEX_ARCH(page), PAGE_WRITE | PAGE_EXEC); \ + } + +#define SETUP_PTL3(ptl2, page, tgt) \ + { \ + SET_PTL3_ADDRESS_ARCH(ptl2, PTL2_INDEX_ARCH(page), (uintptr_t)KA2PA(tgt)); \ + SET_PTL3_FLAGS_ARCH(ptl2, PTL2_INDEX_ARCH(page), PAGE_WRITE | PAGE_EXEC); \ + } + +#define SETUP_FRAME(ptl3, page, tgt) \ + { \ + SET_FRAME_ADDRESS_ARCH(ptl3, PTL3_INDEX_ARCH(page), (uintptr_t)KA2PA(tgt)); \ + SET_FRAME_FLAGS_ARCH(ptl3, PTL3_INDEX_ARCH(page), PAGE_WRITE | PAGE_EXEC); \ + } void page_arch_init(void) { - uintptr_t cur; - unsigned int i; - int identity_flags = PAGE_CACHEABLE | PAGE_EXEC | PAGE_GLOBAL | PAGE_WRITE; - if (config.cpu_active == 1) { + uintptr_t cur; + unsigned int identity_flags = + PAGE_CACHEABLE | PAGE_EXEC | PAGE_GLOBAL | PAGE_WRITE; + page_mapping_operations = &pt_mapping_operations; - + page_table_lock(AS_KERNEL, true); - + /* * PA2KA(identity) mapping for all frames. */ - for (cur = 0; cur < last_frame; cur += FRAME_SIZE) { - /* Standard identity mapping */ + for (cur = 0; cur < last_frame; cur += FRAME_SIZE) page_mapping_insert(AS_KERNEL, PA2KA(cur), cur, identity_flags); - } - - /* Upper kernel mapping - * - from zero to top of kernel (include bottom addresses - * because some are needed for init) - */ - for (cur = PA2KA_CODE(0); cur < config.base + config.kernel_size; cur += FRAME_SIZE) - page_mapping_insert(AS_KERNEL, cur, KA2PA(cur), identity_flags); - - for (cur = config.stack_base; cur < config.stack_base + config.stack_size; cur += FRAME_SIZE) - page_mapping_insert(AS_KERNEL, cur, KA2PA(cur), identity_flags); - - for (i = 0; i < init.cnt; i++) { - for (cur = init.tasks[i].addr; cur < init.tasks[i].addr + init.tasks[i].size; cur += FRAME_SIZE) - page_mapping_insert(AS_KERNEL, PA2KA_CODE(KA2PA(cur)), KA2PA(cur), identity_flags); - } - + page_table_unlock(AS_KERNEL, true); - + exc_register(14, "page_fault", true, (iroutine_t) page_fault); write_cr3((uintptr_t) AS_KERNEL->genarch.page_table); @@ -122,5 +124,4 @@ } - /** Identity page mapper * @@ -128,26 +129,32 @@ * is initializaed. This thing clears page table and fills in the specific * items. + * */ void ident_page_fault(unsigned int n, istate_t *istate) { - uintptr_t page; - static uintptr_t oldpage = 0; - pte_t *aptl_1, *aptl_2, *aptl_3; - - page = read_cr2(); + pte_t *aptl_1; + pte_t *aptl_2; + pte_t *aptl_3; + + uintptr_t page = read_cr2(); + + /* Unmap old address */ if (oldpage) { - /* Unmap old address */ - aptl_1 = PTL1_ADDR(&ptl_0, oldpage); - aptl_2 = PTL2_ADDR(aptl_1, oldpage); - aptl_3 = PTL3_ADDR(aptl_2, oldpage); - + pte_t *aptl_1 = PTL1_ADDR(&ptl_0, oldpage); + pte_t *aptl_2 = PTL2_ADDR(aptl_1, oldpage); + pte_t *aptl_3 = PTL3_ADDR(aptl_2, oldpage); + SET_FRAME_FLAGS_ARCH(aptl_3, PTL3_INDEX_ARCH(oldpage), PAGE_NOT_PRESENT); + if (KA2PA(aptl_3) == KA2PA(helper_ptl3)) SET_PTL3_FLAGS_ARCH(aptl_2, PTL2_INDEX_ARCH(oldpage), PAGE_NOT_PRESENT); + if (KA2PA(aptl_2) == KA2PA(helper_ptl2)) SET_PTL2_FLAGS_ARCH(aptl_1, PTL1_INDEX_ARCH(oldpage), PAGE_NOT_PRESENT); + if (KA2PA(aptl_1) == KA2PA(helper_ptl1)) SET_PTL1_FLAGS_ARCH(&ptl_0, PTL0_INDEX_ARCH(oldpage), PAGE_NOT_PRESENT); } + if (PTL1_PRESENT(&ptl_0, page)) aptl_1 = PTL1_ADDR(&ptl_0, page); @@ -156,5 +163,5 @@ aptl_1 = helper_ptl1; } - + if (PTL2_PRESENT(aptl_1, page)) aptl_2 = PTL2_ADDR(aptl_1, page); @@ -163,5 +170,5 @@ aptl_2 = helper_ptl2; } - + if (PTL3_PRESENT(aptl_2, page)) aptl_3 = PTL3_ADDR(aptl_2, page); @@ -172,18 +179,16 @@ SETUP_FRAME(aptl_3, page, page); - + oldpage = page; } - void page_fault(unsigned int n, istate_t *istate) { - uintptr_t page; - pf_access_t access; - - page = read_cr2(); + uintptr_t page = read_cr2(); if (istate->error_word & PFERR_CODE_RSVD) panic("Reserved bit set in page table entry."); + + pf_access_t access; if (istate->error_word & PFERR_CODE_RW) @@ -195,25 +200,24 @@ if (as_page_fault(page, access, istate) == AS_PF_FAULT) { - fault_if_from_uspace(istate, "Page fault: %#x.", page); - + fault_if_from_uspace(istate, "Page fault: %p.", page); decode_istate(n, istate); - printf("Page fault address: %llx.\n", page); - panic("Page fault."); - } -} - + panic("Page fault: %p", page); + } +} uintptr_t hw_map(uintptr_t physaddr, size_t size) { if (last_frame + ALIGN_UP(size, PAGE_SIZE) > KA2PA(KERNEL_ADDRESS_SPACE_END_ARCH)) - panic("Unable to map physical memory %p (%d bytes).", physaddr, + panic("Unable to map physical memory %p (%" PRIs " bytes).", physaddr, size); uintptr_t virtaddr = PA2KA(last_frame); pfn_t i; - + page_table_lock(AS_KERNEL, true); + for (i = 0; i < ADDR2PFN(ALIGN_UP(size, PAGE_SIZE)); i++) page_mapping_insert(AS_KERNEL, virtaddr + PFN2ADDR(i), physaddr + PFN2ADDR(i), PAGE_NOT_CACHEABLE | PAGE_WRITE); + page_table_unlock(AS_KERNEL, true); Index: kernel/arch/amd64/src/smp/ap.S =================================================================== --- kernel/arch/amd64/src/smp/ap.S (revision 33dac7d46d9290e712c0f579d477a3c7c97536a5) +++ kernel/arch/amd64/src/smp/ap.S (revision a1f60f34d64053b65db351e780b9315fdd0384a3) @@ -55,12 +55,12 @@ xorw %ax, %ax movw %ax, %ds - - lgdtl ap_gdtr # initialize Global Descriptor Table register + + lgdtl ap_gdtr # initialize Global Descriptor Table register movl %cr0, %eax orl $1, %eax - movl %eax, %cr0 # switch to protected mode + movl %eax, %cr0 # switch to protected mode jmpl $gdtselector(KTEXT32_DES), $jump_to_kernel - BOOT_OFFSET + AP_BOOT_OFFSET - + jump_to_kernel: .code32 @@ -72,5 +72,5 @@ movw %ax, %gs - # Enable 64-bit page transaltion entries - CR4.PAE = 1. + # Enable 64-bit page transaltion entries (CR4.PAE = 1). # Paging is not enabled until after long mode is enabled @@ -78,15 +78,15 @@ btsl $5, %eax movl %eax, %cr4 - + leal ptl_0, %eax movl %eax, %cr3 # Enable long mode - movl $EFER_MSR_NUM, %ecx # EFER MSR number - rdmsr # Read EFER - btsl $AMD_LME_FLAG, %eax # Set LME=1 - wrmsr # Write EFER + movl $EFER_MSR_NUM, %ecx # EFER MSR number + rdmsr # Read EFER + btsl $AMD_LME_FLAG, %eax # Set LME=1 + wrmsr # Write EFER - # Enable paging to activate long mode (set CR0.PG=1) + # Enable paging to activate long mode (set CR0.PG = 1) movl %cr0, %eax btsl $31, %eax @@ -98,8 +98,12 @@ .code64 start64: - movq (ctx), %rsp + movabsq $ctx, %rsp + movq (%rsp), %rsp + pushq $0 movq %rsp, %rbp - call main_ap - AP_BOOT_OFFSET + BOOT_OFFSET # never returns + + movabsq $main_ap, %rax + callq *%rax # never returns #endif /* CONFIG_SMP */