Index: kernel/arch/amd64/Makefile.inc =================================================================== --- kernel/arch/amd64/Makefile.inc (revision d6f9fff8ee83683a0df7000a796720690d276858) +++ kernel/arch/amd64/Makefile.inc (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -57,4 +57,6 @@ arch/$(KARCH)/src/debug/stacktrace_asm.S \ arch/$(KARCH)/src/pm.c \ + arch/$(KARCH)/src/vreg.c \ + arch/$(KARCH)/src/kseg.c \ arch/$(KARCH)/src/context.S \ arch/$(KARCH)/src/ddi/ddi.c \ @@ -90,3 +92,4 @@ ARCH_AUTOGENS_AG = \ arch/$(KARCH)/include/arch/istate_struct.ag \ - arch/$(KARCH)/include/arch/context_struct.ag + arch/$(KARCH)/include/arch/context_struct.ag \ + arch/$(KARCH)/include/arch/kseg_struct.ag Index: kernel/arch/amd64/include/arch/asm.h =================================================================== --- kernel/arch/amd64/include/arch/asm.h (revision d6f9fff8ee83683a0df7000a796720690d276858) +++ kernel/arch/amd64/include/arch/asm.h (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -206,12 +206,4 @@ } else *port = val; -} - -/** Swap Hidden part of GS register with visible one */ -NO_TRACE static inline void swapgs(void) -{ - asm volatile ( - "swapgs" - ); } Index: kernel/arch/amd64/include/arch/context_struct.ag =================================================================== --- kernel/arch/amd64/include/arch/context_struct.ag (revision d6f9fff8ee83683a0df7000a796720690d276858) +++ kernel/arch/amd64/include/arch/context_struct.ag (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -75,4 +75,8 @@ type : uint64_t }, + { + name : tp, + type : uint64_t + }, { Index: kernel/arch/amd64/include/arch/cpu.h =================================================================== --- kernel/arch/amd64/include/arch/cpu.h (revision d6f9fff8ee83683a0df7000a796720690d276858) +++ kernel/arch/amd64/include/arch/cpu.h (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -56,9 +56,10 @@ /* MSR registers */ -#define AMD_MSR_STAR 0xc0000081 -#define AMD_MSR_LSTAR 0xc0000082 -#define AMD_MSR_SFMASK 0xc0000084 -#define AMD_MSR_FS 0xc0000100 -#define AMD_MSR_GS 0xc0000101 +#define AMD_MSR_STAR 0xc0000081 +#define AMD_MSR_LSTAR 0xc0000082 +#define AMD_MSR_SFMASK 0xc0000084 +#define AMD_MSR_FS 0xc0000100 +#define AMD_MSR_GS 0xc0000101 +#define AMD_MSR_GS_KERNEL 0xc0000102 #ifndef __ASM__ Index: kernel/arch/amd64/include/arch/kseg.h =================================================================== --- kernel/arch/amd64/include/arch/kseg.h (revision 1a5eca452dd19104f8673b755943e0707abc00e4) +++ kernel/arch/amd64/include/arch/kseg.h (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016 Jakub Jermar + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup amd64 + * @{ + */ +/** @file + */ + +#ifndef KERN_amd64_KSEG_H_ +#define KERN_amd64_KSEG_H_ + +extern void kseg_init(void); + +#endif + +/** @} + */ Index: kernel/arch/amd64/include/arch/kseg_struct.ag =================================================================== --- kernel/arch/amd64/include/arch/kseg_struct.ag (revision 1a5eca452dd19104f8673b755943e0707abc00e4) +++ kernel/arch/amd64/include/arch/kseg_struct.ag (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -0,0 +1,51 @@ +# Copyright (c) 2016 Jakub Jermar +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# - The name of the author may not be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +{ + name : kseg, + + includes : [ + { + include : + } + ], + + members : [ + { + name : ustack_rsp, + type : uint64_t + }, + { + name : kstack_rsp, + type : uint64_t + }, + { + name : fsbase, + type : uint64_t + } + ] +} Index: kernel/arch/amd64/include/arch/proc/thread.h =================================================================== --- kernel/arch/amd64/include/arch/proc/thread.h (revision d6f9fff8ee83683a0df7000a796720690d276858) +++ kernel/arch/amd64/include/arch/proc/thread.h (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -36,12 +36,8 @@ #define KERN_amd64_THREAD_H_ -/* CAUTION: keep these in sync with low level assembly code in syscall_entry */ -#define SYSCALL_USTACK_RSP 0 -#define SYSCALL_KSTACK_RSP 1 +#include typedef struct { - sysarg_t tls; - /** User and kernel RSP for syscalls. */ - uint64_t syscall_rsp[2]; + uint64_t kstack_rsp; } thread_arch_t; Index: kernel/arch/amd64/include/arch/vreg.h =================================================================== --- kernel/arch/amd64/include/arch/vreg.h (revision 1a5eca452dd19104f8673b755943e0707abc00e4) +++ kernel/arch/amd64/include/arch/vreg.h (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2016 Jakub Jermar + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup amd64 + * @{ + */ +/** @file + */ + +#ifndef KERN_amd64_VREG_H_ +#define KERN_amd64_VREG_H_ + +#define VREG_TP 0 + +#ifndef __ASM__ + +#include + +extern uint64_t *vreg_ptr; + +extern void vreg_init(void); + +#endif /* __ASM__ */ + +#endif + +/** @} + */ Index: kernel/arch/amd64/src/amd64.c =================================================================== --- kernel/arch/amd64/src/amd64.c (revision d6f9fff8ee83683a0df7000a796720690d276858) +++ kernel/arch/amd64/src/amd64.c (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -56,4 +56,7 @@ #include #include +#include +#include +#include #ifdef CONFIG_SMP @@ -139,4 +142,7 @@ void arch_post_mm_init(void) { + vreg_init(); + kseg_init(); + if (config.cpu_active == 1) { /* Initialize IRQ routing */ @@ -272,7 +278,4 @@ sysarg_t sys_tls_set(uintptr_t addr) { - THREAD->arch.tls = addr; - write_msr(AMD_MSR_FS, addr); - return EOK; } Index: kernel/arch/amd64/src/asm.S =================================================================== --- kernel/arch/amd64/src/asm.S (revision d6f9fff8ee83683a0df7000a796720690d276858) +++ kernel/arch/amd64/src/asm.S (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -31,4 +31,6 @@ #include #include +#include +#include .text @@ -178,5 +180,5 @@ subq $(ISTATE_SOFT_SIZE + 8), %rsp .endif - + /* * Save the general purpose registers. @@ -199,9 +201,24 @@ /* + * Is this trap from the kernel? + */ + cmpq $(GDT_SELECTOR(KTEXT_DES)), ISTATE_OFFSET_CS(%rsp) + jz 0f + + /* + * Switch to kernel FS base. + */ + swapgs + movl $AMD_MSR_FS, %ecx + movl %gs:KSEG_OFFSET_FSBASE, %eax + movl %gs:KSEG_OFFSET_FSBASE+4, %edx + wrmsr + swapgs + + /* * Imitate a regular stack frame linkage. * Stop stack traces here if we came from userspace. */ - xorl %edx, %edx - cmpq $(GDT_SELECTOR(KTEXT_DES)), ISTATE_OFFSET_CS(%rsp) +0: movl $0x0, %edx cmovnzq %rdx, %rbp @@ -272,12 +289,7 @@ swapgs - /* - * %gs:0 Scratch space for this thread's user RSP - * %gs:8 Address to be used as this thread's kernel RSP - */ - - movq %rsp, %gs:0 /* save this thread's user RSP */ - movq %gs:8, %rsp /* set this thread's kernel RSP */ - + movq %rsp, %gs:KSEG_OFFSET_USTACK_RSP /* save this thread's user RSP */ + movq %gs:KSEG_OFFSET_KSTACK_RSP, %rsp /* set this thread's kernel RSP */ + /* * Note that the space needed for the imitated istate structure has been @@ -308,8 +320,17 @@ /* + * Switch to kernel FS base. + */ + movl $AMD_MSR_FS, %ecx + movl %gs:KSEG_OFFSET_FSBASE, %eax + movl %gs:KSEG_OFFSET_FSBASE+4, %edx + wrmsr + movq ISTATE_OFFSET_RDX(%rsp), %rdx /* restore 3rd argument */ + + /* * Save the return address and the userspace stack on locations that * would normally be taken by them. */ - movq %gs:0, %rax + movq %gs:KSEG_OFFSET_USTACK_RSP, %rax movq %rax, ISTATE_OFFSET_RSP(%rsp) movq %rcx, ISTATE_OFFSET_RIP(%rsp) @@ -325,5 +346,5 @@ swapgs sti - + /* Copy the 4th argument where it is expected */ movq %r10, %rcx Index: kernel/arch/amd64/src/context.S =================================================================== --- kernel/arch/amd64/src/context.S (revision d6f9fff8ee83683a0df7000a796720690d276858) +++ kernel/arch/amd64/src/context.S (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -29,4 +29,5 @@ #include #include +#include .text @@ -50,4 +51,8 @@ movq %r14, CONTEXT_OFFSET_R14(%rdi) movq %r15, CONTEXT_OFFSET_R15(%rdi) + + movq vreg_ptr, %rsi + movq %fs:VREG_TP(%rsi), %rsi + movq %rsi, CONTEXT_OFFSET_TP(%rdi) xorl %eax, %eax # context_save returns 1 @@ -72,7 +77,10 @@ movq CONTEXT_OFFSET_SP(%rdi), %rsp # ctx->sp -> %rsp - movq CONTEXT_OFFSET_PC(%rdi), %rdx - + movq CONTEXT_OFFSET_PC(%rdi), %rdx movq %rdx, (%rsp) + + movq CONTEXT_OFFSET_TP(%rdi), %rcx + movq vreg_ptr, %rsi + movq %rcx, %fs:VREG_TP(%rsi) xorl %eax, %eax # context_restore returns 0 Index: kernel/arch/amd64/src/kseg.c =================================================================== --- kernel/arch/amd64/src/kseg.c (revision 1a5eca452dd19104f8673b755943e0707abc00e4) +++ kernel/arch/amd64/src/kseg.c (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016 Jakub Jermar + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup amd64 + * @{ + */ +/** @file + */ + +#include +#include +#include +#include +#include +#include + +/** + * Allocate and initialize a per-CPU structure to be accessible via the + * GS_KERNEL segment register. + **/ +void kseg_init(void) +{ + kseg_t *kseg; + + kseg = (kseg_t *) malloc(sizeof(kseg_t), FRAME_ATOMIC); + if (!kseg) + panic("Cannot allocate kseg."); + + kseg->ustack_rsp = 0; + kseg->kstack_rsp = 0; + kseg->fsbase = read_msr(AMD_MSR_FS); + + write_msr(AMD_MSR_GS_KERNEL, (uintptr_t) kseg); +} + +/** @} + */ Index: kernel/arch/amd64/src/proc/scheduler.c =================================================================== --- kernel/arch/amd64/src/proc/scheduler.c (revision d6f9fff8ee83683a0df7000a796720690d276858) +++ kernel/arch/amd64/src/proc/scheduler.c (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -42,4 +42,5 @@ #include #include +#include /** Perform amd64 specific tasks needed before the new task is run. @@ -55,16 +56,8 @@ void before_thread_runs_arch(void) { - CPU->arch.tss->rsp0 = - (uintptr_t) &THREAD->kstack[STACK_SIZE]; - - /* - * Syscall support. - */ - swapgs(); - write_msr(AMD_MSR_GS, (uintptr_t) THREAD->arch.syscall_rsp); - swapgs(); - - /* TLS support - set FS to thread local storage */ - write_msr(AMD_MSR_FS, THREAD->arch.tls); + CPU->arch.tss->rsp0 = (uintptr_t) &THREAD->kstack[STACK_SIZE]; + + kseg_t *kseg = (kseg_t *) read_msr(AMD_MSR_GS_KERNEL); + kseg->kstack_rsp = THREAD->arch.kstack_rsp; } Index: kernel/arch/amd64/src/proc/thread.c =================================================================== --- kernel/arch/amd64/src/proc/thread.c (revision d6f9fff8ee83683a0df7000a796720690d276858) +++ kernel/arch/amd64/src/proc/thread.c (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -35,4 +35,5 @@ #include #include +#include /** Perform amd64 specific thread initialization. @@ -43,11 +44,8 @@ void thread_create_arch(thread_t *thread) { - thread->arch.tls = 0; - thread->arch.syscall_rsp[SYSCALL_USTACK_RSP] = 0; - /* * Kernel RSP can be precalculated at thread creation time. */ - thread->arch.syscall_rsp[SYSCALL_KSTACK_RSP] = + thread->arch.kstack_rsp = (uintptr_t) &thread->kstack[PAGE_SIZE - sizeof(istate_t)]; } Index: kernel/arch/amd64/src/vreg.c =================================================================== --- kernel/arch/amd64/src/vreg.c (revision 1a5eca452dd19104f8673b755943e0707abc00e4) +++ kernel/arch/amd64/src/vreg.c (revision 1a5eca452dd19104f8673b755943e0707abc00e4) @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2016 Jakub Jermar + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup amd64 + * @{ + */ +/** @file + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * During initialization, we need to make sure that context_save() and + * context_restore() touch some meaningful address when saving/restoring VREGs. + * When a processor is initialized, we set its FS base to a private page and + * vreg_ptr to zero. + */ +static uint64_t vreg_tp_dummy; +uint64_t *vreg_ptr = &vreg_tp_dummy; + +/** + * Allocate and initialize a per-CPU user page to be accessible via the FS + * segment register and to hold the virtual registers. + */ +void vreg_init(void) +{ + uintptr_t frame; + uint64_t *page; + + frame = frame_alloc(1, FRAME_ATOMIC | FRAME_HIGHMEM, 0); + if (!frame) + frame = frame_alloc(1, FRAME_ATOMIC | FRAME_LOWMEM, 0); + if (!frame) + panic("Cannot allocate VREG frame."); + + page = (uint64_t *) km_map(frame, PAGE_SIZE, + PAGE_READ | PAGE_WRITE | PAGE_USER | PAGE_CACHEABLE); + + write_msr(AMD_MSR_FS, (uintptr_t) page); + + vreg_ptr = NULL; +} + +/** @} + */