Index: kernel/arch/ia32/Makefile.inc =================================================================== --- kernel/arch/ia32/Makefile.inc (revision 3da166f05b0e0513fe18f82b34f75ef99bef955a) +++ kernel/arch/ia32/Makefile.inc (revision 235d31de52ec23ea2a4cb3b7837814333820b9cd) @@ -85,4 +85,5 @@ arch/$(KARCH)/src/smp/mps.c \ arch/$(KARCH)/src/smp/smp.c \ + arch/$(KARCH)/src/smp/smp_call.c \ arch/$(KARCH)/src/atomic.S \ arch/$(KARCH)/src/smp/ipi.c \ Index: kernel/arch/ia32/include/arch/atomic.h =================================================================== --- kernel/arch/ia32/include/arch/atomic.h (revision 3da166f05b0e0513fe18f82b34f75ef99bef955a) +++ kernel/arch/ia32/include/arch/atomic.h (revision 235d31de52ec23ea2a4cb3b7837814333820b9cd) @@ -1,4 +1,5 @@ /* * Copyright (c) 2001-2004 Jakub Jermar + * Copyright (c) 2012 Adam Hraska * All rights reserved. * @@ -113,4 +114,5 @@ } + /** ia32 specific fast spinlock */ NO_TRACE static inline void atomic_lock_arch(atomic_t *val) @@ -142,4 +144,106 @@ } + +#define _atomic_cas_impl(pptr, exp_val, new_val, old_val, prefix) \ +({ \ + switch (sizeof(typeof(*(pptr)))) { \ + case 1: \ + asm volatile ( \ + prefix " cmpxchgb %[newval], %[ptr]\n" \ + : /* Output operands. */ \ + /* Old/current value is returned in eax. */ \ + [oldval] "=a" (old_val), \ + /* (*ptr) will be read and written to, hence "+" */ \ + [ptr] "+m" (*pptr) \ + : /* Input operands. */ \ + /* Expected value must be in eax. */ \ + [expval] "a" (exp_val), \ + /* The new value may be in any register. */ \ + [newval] "r" (new_val) \ + : "memory" \ + ); \ + break; \ + case 2: \ + asm volatile ( \ + prefix " cmpxchgw %[newval], %[ptr]\n" \ + : /* Output operands. */ \ + /* Old/current value is returned in eax. */ \ + [oldval] "=a" (old_val), \ + /* (*ptr) will be read and written to, hence "+" */ \ + [ptr] "+m" (*pptr) \ + : /* Input operands. */ \ + /* Expected value must be in eax. */ \ + [expval] "a" (exp_val), \ + /* The new value may be in any register. */ \ + [newval] "r" (new_val) \ + : "memory" \ + ); \ + break; \ + case 4: \ + asm volatile ( \ + prefix " cmpxchgl %[newval], %[ptr]\n" \ + : /* Output operands. */ \ + /* Old/current value is returned in eax. */ \ + [oldval] "=a" (old_val), \ + /* (*ptr) will be read and written to, hence "+" */ \ + [ptr] "+m" (*pptr) \ + : /* Input operands. */ \ + /* Expected value must be in eax. */ \ + [expval] "a" (exp_val), \ + /* The new value may be in any register. */ \ + [newval] "r" (new_val) \ + : "memory" \ + ); \ + break; \ + } \ +}) + + +#ifndef local_atomic_cas + +#define local_atomic_cas(pptr, exp_val, new_val) \ +({ \ + /* Use proper types and avoid name clashes */ \ + typeof(*(pptr)) _old_val_cas; \ + typeof(*(pptr)) _exp_val_cas = exp_val; \ + typeof(*(pptr)) _new_val_cas = new_val; \ + _atomic_cas_impl(pptr, _exp_val_cas, _new_val_cas, _old_val_cas, ""); \ + \ + _old_val_cas; \ +}) + +#else +/* Check if arch/atomic.h does not accidentally include /atomic.h .*/ +#error Architecture specific cpu local atomics already defined! Check your includes. +#endif + + +#ifndef local_atomic_exchange +/* + * Issuing a xchg instruction always implies lock prefix semantics. + * Therefore, it is cheaper to use a cmpxchg without a lock prefix + * in a loop. + */ +#define local_atomic_exchange(pptr, new_val) \ +({ \ + /* Use proper types and avoid name clashes */ \ + typeof(*(pptr)) _exp_val_x; \ + typeof(*(pptr)) _old_val_x; \ + typeof(*(pptr)) _new_val_x = new_val; \ + \ + do { \ + _exp_val_x = *pptr; \ + _old_val_x = local_atomic_cas(pptr, _exp_val_x, _new_val_x); \ + } while (_old_val_x != _exp_val_x); \ + \ + _old_val_x; \ +}) + +#else +/* Check if arch/atomic.h does not accidentally include /atomic.h .*/ +#error Architecture specific cpu local atomics already defined! Check your includes. +#endif + + #endif Index: kernel/arch/ia32/include/arch/cpu.h =================================================================== --- kernel/arch/ia32/include/arch/cpu.h (revision 3da166f05b0e0513fe18f82b34f75ef99bef955a) +++ kernel/arch/ia32/include/arch/cpu.h (revision 235d31de52ec23ea2a4cb3b7837814333820b9cd) @@ -61,5 +61,7 @@ unsigned int stepping; cpuid_feature_info_t fi; - + + unsigned int id; /** CPU's local, ie physical, APIC ID. */ + tss_t *tss; Index: kernel/arch/ia32/include/arch/interrupt.h =================================================================== --- kernel/arch/ia32/include/arch/interrupt.h (revision 3da166f05b0e0513fe18f82b34f75ef99bef955a) +++ kernel/arch/ia32/include/arch/interrupt.h (revision 235d31de52ec23ea2a4cb3b7837814333820b9cd) @@ -71,4 +71,5 @@ #define VECTOR_TLB_SHOOTDOWN_IPI (IVT_FREEBASE + 1) #define VECTOR_DEBUG_IPI (IVT_FREEBASE + 2) +#define VECTOR_SMP_CALL_IPI (IVT_FREEBASE + 3) extern void (* disable_irqs_function)(uint16_t); Index: kernel/arch/ia32/include/arch/smp/apic.h =================================================================== --- kernel/arch/ia32/include/arch/smp/apic.h (revision 3da166f05b0e0513fe18f82b34f75ef99bef955a) +++ kernel/arch/ia32/include/arch/smp/apic.h (revision 235d31de52ec23ea2a4cb3b7837814333820b9cd) @@ -353,4 +353,5 @@ extern void l_apic_init(void); extern void l_apic_eoi(void); +extern int l_apic_send_custom_ipi(uint8_t, uint8_t); extern int l_apic_broadcast_custom_ipi(uint8_t); extern int l_apic_send_init_ipi(uint8_t); Index: kernel/arch/ia32/src/cpu/cpu.c =================================================================== --- kernel/arch/ia32/src/cpu/cpu.c (revision 3da166f05b0e0513fe18f82b34f75ef99bef955a) +++ kernel/arch/ia32/src/cpu/cpu.c (revision 235d31de52ec23ea2a4cb3b7837814333820b9cd) @@ -160,7 +160,7 @@ void cpu_print_report(cpu_t* cpu) { - printf("cpu%u: (%s family=%u model=%u stepping=%u) %" PRIu16 " MHz\n", - cpu->id, vendor_str[cpu->arch.vendor], cpu->arch.family, - cpu->arch.model, cpu->arch.stepping, cpu->frequency_mhz); + printf("cpu%u: (%s family=%u model=%u stepping=%u apicid=%u) %" PRIu16 + " MHz\n", cpu->id, vendor_str[cpu->arch.vendor], cpu->arch.family, + cpu->arch.model, cpu->arch.stepping, cpu->arch.id, cpu->frequency_mhz); } Index: kernel/arch/ia32/src/ia32.c =================================================================== --- kernel/arch/ia32/src/ia32.c (revision 3da166f05b0e0513fe18f82b34f75ef99bef955a) +++ kernel/arch/ia32/src/ia32.c (revision 235d31de52ec23ea2a4cb3b7837814333820b9cd) @@ -124,5 +124,5 @@ } -void arch_post_cpu_init() +void arch_post_cpu_init(void) { #ifdef CONFIG_SMP Index: kernel/arch/ia32/src/interrupt.c =================================================================== --- kernel/arch/ia32/src/interrupt.c (revision 3da166f05b0e0513fe18f82b34f75ef99bef955a) +++ kernel/arch/ia32/src/interrupt.c (revision 235d31de52ec23ea2a4cb3b7837814333820b9cd) @@ -54,4 +54,6 @@ #include #include +#include +#include /* @@ -170,4 +172,10 @@ tlb_shootdown_ipi_recv(); } + +static void arch_smp_call_ipi_recv(unsigned int n, istate_t *istate) +{ + trap_virtual_eoi(); + smp_call_ipi_recv(); +} #endif @@ -230,4 +238,6 @@ exc_register(VECTOR_TLB_SHOOTDOWN_IPI, "tlb_shootdown", true, (iroutine_t) tlb_shootdown_ipi); + exc_register(VECTOR_SMP_CALL_IPI, "smp_call", true, + (iroutine_t) arch_smp_call_ipi_recv); #endif } Index: kernel/arch/ia32/src/smp/apic.c =================================================================== --- kernel/arch/ia32/src/smp/apic.c (revision 3da166f05b0e0513fe18f82b34f75ef99bef955a) +++ kernel/arch/ia32/src/smp/apic.c (revision 235d31de52ec23ea2a4cb3b7837814333820b9cd) @@ -264,22 +264,44 @@ } -#define DELIVS_PENDING_SILENT_RETRIES 4 - +/* Waits for the destination cpu to accept the previous ipi. */ static void l_apic_wait_for_delivery(void) { icr_t icr; - unsigned retries = 0; - + do { - if (retries++ > DELIVS_PENDING_SILENT_RETRIES) { - retries = 0; -#ifdef CONFIG_DEBUG - log(LF_ARCH, LVL_DEBUG, "IPI is pending."); -#endif - delay(20); - } icr.lo = l_apic[ICRlo]; - } while (icr.delivs == DELIVS_PENDING); - + } while (icr.delivs != DELIVS_IDLE); +} + +/** Send one CPU an IPI vector. + * + * @param apicid Physical APIC ID of the destination CPU. + * @param vector Interrupt vector to be sent. + * + * @return 0 on failure, 1 on success. + */ +int l_apic_send_custom_ipi(uint8_t apicid, uint8_t vector) +{ + icr_t icr; + + /* Wait for a destination cpu to accept our previous ipi. */ + l_apic_wait_for_delivery(); + + icr.lo = l_apic[ICRlo]; + icr.hi = l_apic[ICRhi]; + + icr.delmod = DELMOD_FIXED; + icr.destmod = DESTMOD_PHYS; + icr.level = LEVEL_ASSERT; + icr.shorthand = SHORTHAND_NONE; + icr.trigger_mode = TRIGMOD_LEVEL; + icr.vector = vector; + icr.dest = apicid; + + /* Send the IPI by writing to l_apic[ICRlo]. */ + l_apic[ICRhi] = icr.hi; + l_apic[ICRlo] = icr.lo; + + return apic_poll_errors(); } @@ -294,4 +316,7 @@ { icr_t icr; + + /* Wait for a destination cpu to accept our previous ipi. */ + l_apic_wait_for_delivery(); icr.lo = l_apic[ICRlo]; @@ -304,6 +329,4 @@ l_apic[ICRlo] = icr.lo; - - l_apic_wait_for_delivery(); return apic_poll_errors(); Index: kernel/arch/ia32/src/smp/smp.c =================================================================== --- kernel/arch/ia32/src/smp/smp.c (revision 3da166f05b0e0513fe18f82b34f75ef99bef955a) +++ kernel/arch/ia32/src/smp/smp.c (revision 235d31de52ec23ea2a4cb3b7837814333820b9cd) @@ -55,4 +55,5 @@ #include #include +#include #ifdef CONFIG_SMP @@ -77,4 +78,14 @@ io_apic = (uint32_t *) km_map((uintptr_t) io_apic, PAGE_SIZE, PAGE_WRITE | PAGE_NOT_CACHEABLE); + } +} + +static void cpu_arch_id_init(void) +{ + ASSERT(ops != NULL); + ASSERT(cpus != NULL); + + for (unsigned int i = 0; i < config.cpu_count; ++i) { + cpus[i].arch.id = ops->cpu_apic_id(i); } } @@ -92,4 +103,10 @@ ASSERT(ops != NULL); + + /* + * SMP initialized, cpus array allocated. Assign each CPU its + * physical APIC ID. + */ + cpu_arch_id_init(); /* Index: kernel/arch/ia32/src/smp/smp_call.c =================================================================== --- kernel/arch/ia32/src/smp/smp_call.c (revision 235d31de52ec23ea2a4cb3b7837814333820b9cd) +++ kernel/arch/ia32/src/smp/smp_call.c (revision 235d31de52ec23ea2a4cb3b7837814333820b9cd) @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012 Adam Hraska + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup ia32 + * @{ + */ +/** @file + */ + +#include +#include +#include +#include + +#ifdef CONFIG_SMP + +void arch_smp_call_ipi(unsigned int cpu_id) +{ + (void) l_apic_send_custom_ipi(cpus[cpu_id].arch.id, VECTOR_SMP_CALL_IPI); +} + +#endif /* CONFIG_SMP */ + +/** @} + */