Changeset b1c57a8 in mainline for kernel/arch

Timestamp:

2014-10-09T15:03:55Z (11 years ago)

Author:

Jakub Jermar <jakub@…>

Branches:

lfn, master, serial, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export

Children:

e367939c

Parents:

21799398 (diff), 207e8880 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge from lp:~adam-hraska+lp/helenos/rcu/.

Only merge from the feature branch and resolve all conflicts.

Location:

kernel/arch

Files:

• amd64/Makefile.inc (modified) (1 diff)
• amd64/include/arch/atomic.h (modified) (2 diffs)
• amd64/include/arch/cpu.h (modified) (1 diff)
• amd64/include/arch/interrupt.h (modified) (1 diff)
• amd64/src/amd64.c (modified) (1 diff)
• amd64/src/cpu/cpu.c (modified) (1 diff)
• amd64/src/interrupt.c (modified) (3 diffs)
• amd64/src/smp/smp_call.c (added)
• arm32/Makefile.inc (modified) (1 diff)
• arm32/src/atomic.c (added)
• ia32/Makefile.inc (modified) (1 diff)
• ia32/include/arch/atomic.h (modified) (3 diffs)
• ia32/include/arch/cpu.h (modified) (1 diff)
• ia32/include/arch/interrupt.h (modified) (1 diff)
• ia32/include/arch/smp/apic.h (modified) (1 diff)
• ia32/src/cpu/cpu.c (modified) (1 diff)
• ia32/src/ia32.c (modified) (1 diff)
• ia32/src/interrupt.c (modified) (3 diffs)
• ia32/src/smp/apic.c (modified) (3 diffs)
• ia32/src/smp/smp.c (modified) (3 diffs)
• ia32/src/smp/smp_call.c (added)
• ia64/Makefile.inc (modified) (1 diff)
• ia64/src/smp/smp_call.c (added)
• mips32/Makefile.inc (modified) (1 diff)
• mips32/src/smp/smp_call.c (added)
• sparc64/Makefile.inc (modified) (1 diff)
• sparc64/include/arch/barrier.h (modified) (1 diff)
• sparc64/include/arch/interrupt.h (modified) (1 diff)
• sparc64/include/arch/smp/sun4u/ipi.h (added)
• sparc64/src/debug/stacktrace.c (modified) (1 diff)
• sparc64/src/smp/sun4u/ipi.c (modified) (3 diffs)
• sparc64/src/smp/sun4u/smp_call.c (added)

kernel/arch/amd64/Makefile.inc

@@ -84 +84 @@
                 arch/$(KARCH)/src/smp/ipi.c \
                 arch/$(KARCH)/src/smp/mps.c \
+                arch/$(KARCH)/src/smp/smp_call.c \
                 arch/$(KARCH)/src/smp/smp.c
 endif

kernel/arch/amd64/include/arch/atomic.h

@@ -1 +1 @@
 /*
  * Copyright (c) 2001-2004 Jakub Jermar
+ * Copyright (c) 2012      Adam Hraska
  * All rights reserved.
  *
@@ -140 +141 @@
 }
 
+
+#define _atomic_cas_impl(pptr, exp_val, new_val, old_val, prefix) \
+({ \
+        switch (sizeof(typeof(*(pptr)))) { \
+        case 1: \
+                asm volatile ( \
+                        prefix " cmpxchgb %[newval], %[ptr]\n" \
+                        : /* Output operands. */ \
+                        /* Old/current value is returned in eax. */ \
+                        [oldval] "=a" (old_val), \
+                        /* (*ptr) will be read and written to, hence "+" */ \
+                        [ptr] "+m" (*pptr) \
+                        : /* Input operands. */ \
+                        /* Expected value must be in eax. */ \
+                        [expval] "a" (exp_val), \
+                        /* The new value may be in any register. */ \
+                        [newval] "r" (new_val) \
+                        : "memory" \
+                ); \
+                break; \
+        case 2: \
+                asm volatile ( \
+                        prefix " cmpxchgw %[newval], %[ptr]\n" \
+                        : /* Output operands. */ \
+                        /* Old/current value is returned in eax. */ \
+                        [oldval] "=a" (old_val), \
+                        /* (*ptr) will be read and written to, hence "+" */ \
+                        [ptr] "+m" (*pptr) \
+                        : /* Input operands. */ \
+                        /* Expected value must be in eax. */ \
+                        [expval] "a" (exp_val), \
+                        /* The new value may be in any register. */ \
+                        [newval] "r" (new_val) \
+                        : "memory" \
+                ); \
+                break; \
+        case 4: \
+                asm volatile ( \
+                        prefix " cmpxchgl %[newval], %[ptr]\n" \
+                        : /* Output operands. */ \
+                        /* Old/current value is returned in eax. */ \
+                        [oldval] "=a" (old_val), \
+                        /* (*ptr) will be read and written to, hence "+" */ \
+                        [ptr] "+m" (*pptr) \
+                        : /* Input operands. */ \
+                        /* Expected value must be in eax. */ \
+                        [expval] "a" (exp_val), \
+                        /* The new value may be in any register. */ \
+                        [newval] "r" (new_val) \
+                        : "memory" \
+                ); \
+                break; \
+        case 8: \
+                asm volatile ( \
+                        prefix " cmpxchgq %[newval], %[ptr]\n" \
+                        : /* Output operands. */ \
+                        /* Old/current value is returned in eax. */ \
+                        [oldval] "=a" (old_val), \
+                        /* (*ptr) will be read and written to, hence "+" */ \
+                        [ptr] "+m" (*pptr) \
+                        : /* Input operands. */ \
+                        /* Expected value must be in eax. */ \
+                        [expval] "a" (exp_val), \
+                        /* The new value may be in any register. */ \
+                        [newval] "r" (new_val) \
+                        : "memory" \
+                ); \
+                break; \
+        } \
+})
+
+
+#ifndef local_atomic_cas
+
+#define local_atomic_cas(pptr, exp_val, new_val) \
+({ \
+        /* Use proper types and avoid name clashes */ \
+        typeof(*(pptr)) _old_val_cas; \
+        typeof(*(pptr)) _exp_val_cas = exp_val; \
+        typeof(*(pptr)) _new_val_cas = new_val; \
+        _atomic_cas_impl(pptr, _exp_val_cas, _new_val_cas, _old_val_cas, ""); \
+        \
+        _old_val_cas; \
+})
+
+#else
+/* Check if arch/atomic.h does not accidentally include /atomic.h .*/
+#error Architecture specific cpu local atomics already defined! Check your includes.
 #endif
 
+
+#ifndef local_atomic_exchange
+/* 
+ * Issuing a xchg instruction always implies lock prefix semantics.
+ * Therefore, it is cheaper to use a cmpxchg without a lock prefix 
+ * in a loop.
+ */
+#define local_atomic_exchange(pptr, new_val) \
+({ \
+        /* Use proper types and avoid name clashes */ \
+        typeof(*(pptr)) _exp_val_x; \
+        typeof(*(pptr)) _old_val_x; \
+        typeof(*(pptr)) _new_val_x = new_val; \
+        \
+        do { \
+                _exp_val_x = *pptr; \
+                _old_val_x = local_atomic_cas(pptr, _exp_val_x, _new_val_x); \
+        } while (_old_val_x != _exp_val_x); \
+        \
+        _old_val_x; \
+})
+
+#else
+/* Check if arch/atomic.h does not accidentally include /atomic.h .*/
+#error Architecture specific cpu local atomics already defined! Check your includes.
+#endif
+
+
+#endif
+
 /** @}
  */

kernel/arch/amd64/include/arch/cpu.h

@@ -73 +73 @@
         tss_t *tss;
         
+        unsigned int id; /** CPU's local, ie physical, APIC ID. */
+        
         size_t iomapver_copy;  /** Copy of TASK's I/O Permission bitmap generation count. */
 } cpu_arch_t;

kernel/arch/amd64/include/arch/interrupt.h

@@ -69 +69 @@
 #define VECTOR_TLB_SHOOTDOWN_IPI  (IVT_FREEBASE + 1)
 #define VECTOR_DEBUG_IPI          (IVT_FREEBASE + 2)
+#define VECTOR_SMP_CALL_IPI       (IVT_FREEBASE + 3)
 
 extern void (* disable_irqs_function)(uint16_t);

kernel/arch/amd64/src/amd64.c

@@ -168 +168 @@
 }
 
-void arch_post_cpu_init()
+void arch_post_cpu_init(void)
 {
 #ifdef CONFIG_SMP

kernel/arch/amd64/src/cpu/cpu.c

@@ -158 +158 @@
 void cpu_print_report(cpu_t* m)
 {
-        printf("cpu%d: (%s family=%d model=%d stepping=%d) %dMHz\n",
+        printf("cpu%d: (%s family=%d model=%d stepping=%d apicid=%u) %dMHz\n",
             m->id, vendor_str[m->arch.vendor], m->arch.family, m->arch.model,
-            m->arch.stepping, m->frequency_mhz);
+            m->arch.stepping, m->arch.id, m->frequency_mhz);
 }
 

kernel/arch/amd64/src/interrupt.c

@@ -55 +55 @@
 #include <symtab.h>
 #include <stacktrace.h>
+#include <smp/smp_call.h>
 
 /*
@@ -161 +162 @@
         trap_virtual_eoi();
         tlb_shootdown_ipi_recv();
+}
+
+static void arch_smp_call_ipi_recv(unsigned int n, istate_t *istate)
+{
+        trap_virtual_eoi();
+        smp_call_ipi_recv();
 }
 #endif
@@ -224 +231 @@
         exc_register(VECTOR_TLB_SHOOTDOWN_IPI, "tlb_shootdown", true,
             (iroutine_t) tlb_shootdown_ipi);
+        exc_register(VECTOR_SMP_CALL_IPI, "smp_call", true, 
+                (iroutine_t) arch_smp_call_ipi_recv);
 #endif
 }

kernel/arch/arm32/Makefile.inc

@@ -65 +65 @@
         arch/$(KARCH)/src/mm/tlb.c \
         arch/$(KARCH)/src/mm/page_fault.c \
+        arch/$(KARCH)/src/atomic.c \
         arch/$(KARCH)/src/ras.c
 

kernel/arch/ia32/Makefile.inc

@@ -85 +85 @@
         arch/$(KARCH)/src/smp/mps.c \
         arch/$(KARCH)/src/smp/smp.c \
+        arch/$(KARCH)/src/smp/smp_call.c \
         arch/$(KARCH)/src/atomic.S \
         arch/$(KARCH)/src/smp/ipi.c \

kernel/arch/ia32/include/arch/atomic.h

@@ -1 +1 @@
 /*
  * Copyright (c) 2001-2004 Jakub Jermar
+ * Copyright (c) 2012      Adam Hraska
  * All rights reserved.
  *
@@ -113 +114 @@
 }
 
+
 /** ia32 specific fast spinlock */
 NO_TRACE static inline void atomic_lock_arch(atomic_t *val)
@@ -142 +144 @@
 }
 
+
+#define _atomic_cas_impl(pptr, exp_val, new_val, old_val, prefix) \
+({ \
+        switch (sizeof(typeof(*(pptr)))) { \
+        case 1: \
+                asm volatile ( \
+                        prefix " cmpxchgb %[newval], %[ptr]\n" \
+                        : /* Output operands. */ \
+                        /* Old/current value is returned in eax. */ \
+                        [oldval] "=a" (old_val), \
+                        /* (*ptr) will be read and written to, hence "+" */ \
+                        [ptr] "+m" (*pptr) \
+                        : /* Input operands. */ \
+                        /* Expected value must be in eax. */ \
+                        [expval] "a" (exp_val), \
+                        /* The new value may be in any register. */ \
+                        [newval] "r" (new_val) \
+                        : "memory" \
+                ); \
+                break; \
+        case 2: \
+                asm volatile ( \
+                        prefix " cmpxchgw %[newval], %[ptr]\n" \
+                        : /* Output operands. */ \
+                        /* Old/current value is returned in eax. */ \
+                        [oldval] "=a" (old_val), \
+                        /* (*ptr) will be read and written to, hence "+" */ \
+                        [ptr] "+m" (*pptr) \
+                        : /* Input operands. */ \
+                        /* Expected value must be in eax. */ \
+                        [expval] "a" (exp_val), \
+                        /* The new value may be in any register. */ \
+                        [newval] "r" (new_val) \
+                        : "memory" \
+                ); \
+                break; \
+        case 4: \
+                asm volatile ( \
+                        prefix " cmpxchgl %[newval], %[ptr]\n" \
+                        : /* Output operands. */ \
+                        /* Old/current value is returned in eax. */ \
+                        [oldval] "=a" (old_val), \
+                        /* (*ptr) will be read and written to, hence "+" */ \
+                        [ptr] "+m" (*pptr) \
+                        : /* Input operands. */ \
+                        /* Expected value must be in eax. */ \
+                        [expval] "a" (exp_val), \
+                        /* The new value may be in any register. */ \
+                        [newval] "r" (new_val) \
+                        : "memory" \
+                ); \
+                break; \
+        } \
+})
+
+
+#ifndef local_atomic_cas
+
+#define local_atomic_cas(pptr, exp_val, new_val) \
+({ \
+        /* Use proper types and avoid name clashes */ \
+        typeof(*(pptr)) _old_val_cas; \
+        typeof(*(pptr)) _exp_val_cas = exp_val; \
+        typeof(*(pptr)) _new_val_cas = new_val; \
+        _atomic_cas_impl(pptr, _exp_val_cas, _new_val_cas, _old_val_cas, ""); \
+        \
+        _old_val_cas; \
+})
+
+#else
+/* Check if arch/atomic.h does not accidentally include /atomic.h .*/
+#error Architecture specific cpu local atomics already defined! Check your includes.
+#endif
+
+
+#ifndef local_atomic_exchange
+/* 
+ * Issuing a xchg instruction always implies lock prefix semantics.
+ * Therefore, it is cheaper to use a cmpxchg without a lock prefix 
+ * in a loop.
+ */
+#define local_atomic_exchange(pptr, new_val) \
+({ \
+        /* Use proper types and avoid name clashes */ \
+        typeof(*(pptr)) _exp_val_x; \
+        typeof(*(pptr)) _old_val_x; \
+        typeof(*(pptr)) _new_val_x = new_val; \
+        \
+        do { \
+                _exp_val_x = *pptr; \
+                _old_val_x = local_atomic_cas(pptr, _exp_val_x, _new_val_x); \
+        } while (_old_val_x != _exp_val_x); \
+        \
+        _old_val_x; \
+})
+
+#else
+/* Check if arch/atomic.h does not accidentally include /atomic.h .*/
+#error Architecture specific cpu local atomics already defined! Check your includes.
+#endif
+
+
 #endif
 

kernel/arch/ia32/include/arch/cpu.h

@@ -61 +61 @@
         unsigned int stepping;
         cpuid_feature_info_t fi;
-        
+
+        unsigned int id; /** CPU's local, ie physical, APIC ID. */
+
         tss_t *tss;
         

kernel/arch/ia32/include/arch/interrupt.h

@@ -69 +69 @@
 #define VECTOR_TLB_SHOOTDOWN_IPI  (IVT_FREEBASE + 1)
 #define VECTOR_DEBUG_IPI          (IVT_FREEBASE + 2)
+#define VECTOR_SMP_CALL_IPI       (IVT_FREEBASE + 3)
 
 extern void (* disable_irqs_function)(uint16_t);

kernel/arch/ia32/include/arch/smp/apic.h

@@ -353 +353 @@
 extern void l_apic_init(void);
 extern void l_apic_eoi(void);
+extern int l_apic_send_custom_ipi(uint8_t, uint8_t);
 extern int l_apic_broadcast_custom_ipi(uint8_t);
 extern int l_apic_send_init_ipi(uint8_t);

kernel/arch/ia32/src/cpu/cpu.c

@@ -160 +160 @@
 void cpu_print_report(cpu_t* cpu)
 {
-        printf("cpu%u: (%s family=%u model=%u stepping=%u) %" PRIu16 " MHz\n",
-                cpu->id, vendor_str[cpu->arch.vendor], cpu->arch.family,
-                cpu->arch.model, cpu->arch.stepping, cpu->frequency_mhz);
+        printf("cpu%u: (%s family=%u model=%u stepping=%u apicid=%u) %" PRIu16 
+                " MHz\n", cpu->id, vendor_str[cpu->arch.vendor], cpu->arch.family,
+                cpu->arch.model, cpu->arch.stepping, cpu->arch.id, cpu->frequency_mhz);
 }
 

kernel/arch/ia32/src/ia32.c

@@ -122 +122 @@
 }
 
-void arch_post_cpu_init()
+void arch_post_cpu_init(void)
 {
 #ifdef CONFIG_SMP

kernel/arch/ia32/src/interrupt.c

@@ -54 +54 @@
 #include <symtab.h>
 #include <stacktrace.h>
+#include <smp/smp_call.h>
+#include <proc/task.h>
 
 /*
@@ -170 +172 @@
         tlb_shootdown_ipi_recv();
 }
+
+static void arch_smp_call_ipi_recv(unsigned int n, istate_t *istate)
+{
+        trap_virtual_eoi();
+        smp_call_ipi_recv();
+}
 #endif
 
@@ -230 +238 @@
         exc_register(VECTOR_TLB_SHOOTDOWN_IPI, "tlb_shootdown", true,
             (iroutine_t) tlb_shootdown_ipi);
+        exc_register(VECTOR_SMP_CALL_IPI, "smp_call", true,
+            (iroutine_t) arch_smp_call_ipi_recv);
 #endif
 }

kernel/arch/ia32/src/smp/apic.c

@@ -264 +264 @@
 }
 
-#define DELIVS_PENDING_SILENT_RETRIES   4       
-
+/* Waits for the destination cpu to accept the previous ipi. */
 static void l_apic_wait_for_delivery(void)
 {
         icr_t icr;
-        unsigned retries = 0;
-
+        
         do {
-                if (retries++ > DELIVS_PENDING_SILENT_RETRIES) {
-                        retries = 0;
-#ifdef CONFIG_DEBUG
-                        log(LF_ARCH, LVL_DEBUG, "IPI is pending.");
-#endif
-                        delay(20);
-                }
                 icr.lo = l_apic[ICRlo];
-        } while (icr.delivs == DELIVS_PENDING);
-        
+        } while (icr.delivs != DELIVS_IDLE);
+}
+
+/** Send one CPU an IPI vector.
+ *
+ * @param apicid Physical APIC ID of the destination CPU.
+ * @param vector Interrupt vector to be sent.
+ *
+ * @return 0 on failure, 1 on success.
+ */
+int l_apic_send_custom_ipi(uint8_t apicid, uint8_t vector)
+{
+        icr_t icr;
+
+        /* Wait for a destination cpu to accept our previous ipi. */
+        l_apic_wait_for_delivery();
+        
+        icr.lo = l_apic[ICRlo];
+        icr.hi = l_apic[ICRhi];
+        
+        icr.delmod = DELMOD_FIXED;
+        icr.destmod = DESTMOD_PHYS;
+        icr.level = LEVEL_ASSERT;
+        icr.shorthand = SHORTHAND_NONE;
+        icr.trigger_mode = TRIGMOD_LEVEL;
+        icr.vector = vector;
+        icr.dest = apicid;
+
+        /* Send the IPI by writing to l_apic[ICRlo]. */
+        l_apic[ICRhi] = icr.hi;
+        l_apic[ICRlo] = icr.lo;
+        
+        return apic_poll_errors();
 }
 
@@ -294 +316 @@
 {
         icr_t icr;
+
+        /* Wait for a destination cpu to accept our previous ipi. */
+        l_apic_wait_for_delivery();
         
         icr.lo = l_apic[ICRlo];
@@ -304 +329 @@
         
         l_apic[ICRlo] = icr.lo;
-
-        l_apic_wait_for_delivery();
         
         return apic_poll_errors();

kernel/arch/ia32/src/smp/smp.c

@@ -55 +55 @@
 #include <memstr.h>
 #include <arch/drivers/i8259.h>
+#include <cpu.h>
 
 #ifdef CONFIG_SMP
@@ -77 +78 @@
                 io_apic = (uint32_t *) km_map((uintptr_t) io_apic, PAGE_SIZE,
                     PAGE_WRITE | PAGE_NOT_CACHEABLE);
+        }
+}
+
+static void cpu_arch_id_init(void)
+{
+        ASSERT(ops != NULL);
+        ASSERT(cpus != NULL);
+        
+        for (unsigned int i = 0; i < config.cpu_count; ++i) {
+                cpus[i].arch.id = ops->cpu_apic_id(i);
         }
 }
@@ -92 +103 @@
         
         ASSERT(ops != NULL);
+
+        /*
+         * SMP initialized, cpus array allocated. Assign each CPU its 
+         * physical APIC ID.
+         */
+        cpu_arch_id_init();
         
         /*

kernel/arch/ia64/Makefile.inc

@@ -61 +61 @@
         arch/$(KARCH)/src/ddi/ddi.c \
         arch/$(KARCH)/src/smp/smp.c \
+        arch/$(KARCH)/src/smp/smp_call.c \
         arch/$(KARCH)/src/drivers/it.c
 

kernel/arch/mips32/Makefile.inc

@@ -71 +71 @@
         arch/$(KARCH)/src/fpu_context.c \
         arch/$(KARCH)/src/smp/smp.c \
+        arch/$(KARCH)/src/smp/smp_call.c \
         arch/$(KARCH)/src/machine_func.c
 

kernel/arch/sparc64/Makefile.inc

@@ -100 +100 @@
         ARCH_SOURCES += \
                 arch/$(KARCH)/src/smp/$(USARCH)/smp.c \
+                arch/$(KARCH)/src/smp/$(USARCH)/smp_call.c \
                 arch/$(KARCH)/src/smp/$(USARCH)/ipi.c
 endif

kernel/arch/sparc64/include/arch/barrier.h

@@ -37 +37 @@
 
 #include <trace.h>
+
+#ifdef KERNEL
+#include <arch/common.h>
+#else
+#include <libarch/common.h>
+#endif
 
 /*

kernel/arch/sparc64/include/arch/interrupt.h

@@ -47 +47 @@
 
 enum {
-        IPI_TLB_SHOOTDOWN = VECTOR_TLB_SHOOTDOWN_IPI
+        IPI_TLB_SHOOTDOWN = VECTOR_TLB_SHOOTDOWN_IPI,
+        IPI_SMP_CALL
 };
 

kernel/arch/sparc64/src/debug/stacktrace.c

@@ -36 +36 @@
 #include <syscall/copy.h>
 #include <typedefs.h>
+#include <proc/thread.h>
 
 #include <arch.h>

kernel/arch/sparc64/src/smp/sun4u/ipi.c

@@ -34 +34 @@
 
 #include <smp/ipi.h>
+#include <arch/smp/sun4u/ipi.h>
 #include <cpu.h>
 #include <arch.h>
@@ -40 +41 @@
 #include <config.h>
 #include <mm/tlb.h>
+#include <smp/smp_call.h>
 #include <arch/interrupt.h>
 #include <arch/trap/interrupt.h>
@@ -171 +173 @@
 }
 
+
+/*
+ * Deliver an IPI to the specified processors (except the current one).
+ *
+ * Interrupts must be disabled.
+ *
+ * @param cpu_id Destination cpu id (index into cpus array). Must not 
+ *               be the current cpu.
+ * @param ipi    IPI number.
+ */
+void ipi_unicast_arch(unsigned int cpu_id, int ipi)
+{
+        ASSERT(&cpus[cpu_id] != CPU);
+        
+        if (ipi == IPI_SMP_CALL) {
+                cross_call(cpus[cpu_id].arch.mid, smp_call_ipi_recv);
+        } else {
+                panic("Unknown IPI (%d).\n", ipi);
+                return;
+        }
+}
+
 /** @}
  */