Index: kernel/arch/sparc64/include/cpu.h =================================================================== --- kernel/arch/sparc64/include/cpu.h (revision 0f6a3376dc93b2660c71d378318b088d41afec79) +++ kernel/arch/sparc64/include/cpu.h (revision 5d7daffb996b8fe2ed2955eedceaf948fc1a9f75) @@ -40,4 +40,8 @@ #include +#ifdef CONFIG_SMP +#include +#endif + #define MANUF_FUJITSU 0x04 #define MANUF_ULTRASPARC 0x17 /**< UltraSPARC I, UltraSPARC II */ @@ -54,10 +58,16 @@ typedef struct { - uint32_t mid; /**< Processor ID as read from UPA_CONFIG. */ + uint32_t mid; /**< Processor ID as read from + UPA_CONFIG. */ ver_reg_t ver; uint32_t clock_frequency; /**< Processor frequency in Hz. */ uint64_t next_tick_cmpr; /**< Next clock interrupt should be - generated when the TICK register - matches this value. */ + generated when the TICK register + matches this value. */ +#ifdef CONFIG_SMP + int dcache_active; + dcache_shootdown_msg_t dcache_messages[DCACHE_MSG_QUEUE_LEN]; + count_t dcache_message_count; +#endif } cpu_arch_t; Index: kernel/arch/sparc64/include/interrupt.h =================================================================== --- kernel/arch/sparc64/include/interrupt.h (revision 0f6a3376dc93b2660c71d378318b088d41afec79) +++ kernel/arch/sparc64/include/interrupt.h (revision 5d7daffb996b8fe2ed2955eedceaf948fc1a9f75) @@ -47,5 +47,6 @@ enum { - IPI_TLB_SHOOTDOWN = VECTOR_TLB_SHOOTDOWN_IPI + IPI_TLB_SHOOTDOWN = VECTOR_TLB_SHOOTDOWN_IPI, + IPI_DCACHE_SHOOTDOWN }; Index: kernel/arch/sparc64/include/mm/cache.h =================================================================== --- kernel/arch/sparc64/include/mm/cache.h (revision 0f6a3376dc93b2660c71d378318b088d41afec79) +++ kernel/arch/sparc64/include/mm/cache.h (revision 5d7daffb996b8fe2ed2955eedceaf948fc1a9f75) @@ -44,7 +44,41 @@ dcache_flush_tag(PAGE_COLOR((p)), ADDR2PFN((f))); +/** + * Enumerations to differentiate among different scopes of D-Cache + * invalidation. + */ +typedef enum { + DCACHE_INVL_INVALID, + DCACHE_INVL_ALL, + DCACHE_INVL_COLOR, + DCACHE_INVL_FRAME +} dcache_invalidate_type_t; + +/** + * Number of messages that can be queued in the cpu_arch_t structure at a time. + */ +#define DCACHE_MSG_QUEUE_LEN 10 + +/** D-cache shootdown message type. */ +typedef struct { + dcache_invalidate_type_t type; + int color; + uintptr_t frame; +} dcache_shootdown_msg_t; + extern void dcache_flush(void); extern void dcache_flush_color(int c); extern void dcache_flush_tag(int c, pfn_t tag); + +#ifdef CONFIG_SMP +extern void dcache_shootdown_start(dcache_invalidate_type_t type, int color, + uintptr_t frame); +extern void dcache_shootdown_finalize(void); +extern void dcache_shootdown_ipi_recv(void); +#else +#define dcache_shootdown_start(t, c, f) +#define dcache_shootdown_finalize() +#define dcache_shootdown_ipi_recv() +#endif /* CONFIG_SMP */ #endif Index: kernel/arch/sparc64/src/cpu/cpu.c =================================================================== --- kernel/arch/sparc64/src/cpu/cpu.c (revision 0f6a3376dc93b2660c71d378318b088d41afec79) +++ kernel/arch/sparc64/src/cpu/cpu.c (revision 5d7daffb996b8fe2ed2955eedceaf948fc1a9f75) @@ -52,4 +52,9 @@ CPU->arch.mid = upa_config.mid; +#if (defined(CONFIG_SMP) && defined(CONFIG_VIRT_IDX_DCACHE)) + CPU->arch.dcache_active = 1; + CPU->arch.dcache_message_count = 0; +#endif + /* * Detect processor frequency. Index: kernel/arch/sparc64/src/mm/cache.c =================================================================== --- kernel/arch/sparc64/src/mm/cache.c (revision 0f6a3376dc93b2660c71d378318b088d41afec79) +++ kernel/arch/sparc64/src/mm/cache.c (revision 5d7daffb996b8fe2ed2955eedceaf948fc1a9f75) @@ -32,8 +32,125 @@ /** * @file + * @brief D-cache shootdown algorithm. */ #include +#ifdef CONFIG_SMP +#ifdef CONFIG_VIRT_IDX_DCACHE + +#include +#include +#include +#include +#include + +/** + * This spinlock is used by the processors to synchronize during the D-cache + * shootdown. + */ +SPINLOCK_INITIALIZE(dcachelock); + +/** Initialize the D-cache shootdown sequence. + * + * Start the shootdown sequence by sending out an IPI and wait until all + * processors spin on the dcachelock spinlock. + * + * @param type Scope of the D-cache shootdown. + * @param color Color to be invalidated; applicable only for DCACHE_INVL_COLOR + * and DCACHE_INVL_FRAME invalidation types. + * @param frame Frame to be invalidated; applicable only for DCACHE_INVL_FRAME + * invalidation types. + */ +void dcache_shootdown_start(dcache_invalidate_type_t type, int color, + uintptr_t frame) +{ + int i; + + CPU->arch.dcache_active = 0; + spinlock_lock(&dcachelock); + + for (i = 0; i < config.cpu_count; i++) { + cpu_t *cpu; + + if (i == CPU->id) + continue; + + cpu = &cpus[i]; + spinlock_lock(&cpu->lock); + if (cpu->arch.dcache_message_count == + DCACHE_MSG_QUEUE_LEN) { + /* + * The queue is full, flush the cache entirely. + */ + cpu->arch.dcache_message_count = 1; + cpu->arch.dcache_messages[0].type = DCACHE_INVL_ALL; + cpu->arch.dcache_messages[0].color = 0; /* ignored */ + cpu->arch.dcache_messages[0].frame = 0; /* ignored */ + } else { + index_t idx = cpu->arch.dcache_message_count++; + cpu->arch.dcache_messages[idx].type = type; + cpu->arch.dcache_messages[idx].color = color; + cpu->arch.dcache_messages[idx].frame = frame; + } + spinlock_unlock(&cpu->lock); + } + + ipi_broadcast(IPI_DCACHE_SHOOTDOWN); + +busy_wait: + for (i = 0; i < config.cpu_count; i++) + if (cpus[i].arch.dcache_active) + goto busy_wait; +} + +/** Finish the D-cache shootdown sequence. */ +void dcache_shootdown_finalize(void) +{ + spinlock_unlock(&dcachelock); + CPU->arch.dcache_active = 1; +} + +/** Process the D-cache shootdown IPI. */ +void dcache_shootdown_ipi_recv(void) +{ + int i; + + ASSERT(CPU); + + CPU->arch.dcache_active = 0; + spinlock_lock(&dcachelock); + spinlock_unlock(&dcachelock); + + spinlock_lock(&CPU->lock); + ASSERT(CPU->arch.dcache_message_count < DCACHE_MSG_QUEUE_LEN); + for (i = 0; i < CPU->arch.dcache_message_count; i++) { + switch (CPU->arch.dcache_messages[i].type) { + case DCACHE_INVL_ALL: + dcache_flush(); + goto flushed; + break; + case DCACHE_INVL_COLOR: + dcache_flush_color(CPU->arch.dcache_messages[i].color); + break; + case DCACHE_INVL_FRAME: + dcache_flush_frame(CPU->arch.dcache_messages[i].color, + CPU->arch.dcache_messages[i].frame); + break; + default: + panic("unknown type (%d)\n", + CPU->arch.dcache_messages[i].type); + } + } +flushed: + CPU->arch.dcache_message_count = 0; + spinlock_unlock(&CPU->lock); + + CPU->arch.dcache_active = 1; +} + +#endif /* CONFIG_VIRT_IDX_DCACHE */ +#endif /* CONFIG_SMP */ + /** @} */ Index: kernel/arch/sparc64/src/mm/tlb.c =================================================================== --- kernel/arch/sparc64/src/mm/tlb.c (revision 0f6a3376dc93b2660c71d378318b088d41afec79) +++ kernel/arch/sparc64/src/mm/tlb.c (revision 5d7daffb996b8fe2ed2955eedceaf948fc1a9f75) @@ -483,8 +483,8 @@ for (i = 0; i < cnt; i++) { - itlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY, page + i * - PAGE_SIZE); - dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY, page + i * - PAGE_SIZE); + itlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY, + page + i * PAGE_SIZE); + dtlb_demap(TLB_DEMAP_PAGE, TLB_DEMAP_PRIMARY, + page + i * PAGE_SIZE); } Index: kernel/arch/sparc64/src/smp/ipi.c =================================================================== --- kernel/arch/sparc64/src/smp/ipi.c (revision 0f6a3376dc93b2660c71d378318b088d41afec79) +++ kernel/arch/sparc64/src/smp/ipi.c (revision 5d7daffb996b8fe2ed2955eedceaf948fc1a9f75) @@ -40,4 +40,5 @@ #include #include +#include #include #include @@ -79,7 +80,7 @@ asi_u64_write(ASI_UDB_INTR_W, ASI_UDB_INTR_W_DATA_1, 0); asi_u64_write(ASI_UDB_INTR_W, ASI_UDB_INTR_W_DATA_2, 0); - asi_u64_write(ASI_UDB_INTR_W, (mid << - INTR_VEC_DISPATCH_MID_SHIFT) | ASI_UDB_INTR_W_DISPATCH, - 0); + asi_u64_write(ASI_UDB_INTR_W, + (mid << INTR_VEC_DISPATCH_MID_SHIFT) | + ASI_UDB_INTR_W_DISPATCH, 0); membar(); @@ -125,4 +126,9 @@ func = tlb_shootdown_ipi_recv; break; +#if (defined(CONFIG_SMP) && (defined(CONFIG_VIRT_IDX_DCACHE))) + case IPI_DCACHE_SHOOTDOWN: + func = dcache_shootdown_ipi_recv; + break; +#endif default: panic("Unknown IPI (%d).\n", ipi); Index: kernel/arch/sparc64/src/trap/interrupt.c =================================================================== --- kernel/arch/sparc64/src/trap/interrupt.c (revision 0f6a3376dc93b2660c71d378318b088d41afec79) +++ kernel/arch/sparc64/src/trap/interrupt.c (revision 5d7daffb996b8fe2ed2955eedceaf948fc1a9f75) @@ -45,4 +45,5 @@ #include #include +#include #include #include @@ -84,11 +85,15 @@ /* * This is a cross-call. - * data0 contains address of kernel function. + * data0 contains address of the kernel function. * We call the function only after we verify - * it is on of the supported ones. + * it is one of the supported ones. */ #ifdef CONFIG_SMP if (data0 == (uintptr_t) tlb_shootdown_ipi_recv) { tlb_shootdown_ipi_recv(); +#ifdef CONFIG_VIRT_IDX_DCACHE + } else if (data0 == (uintptr_t) dcache_shootdown_ipi_recv) { + dcache_shootdown_ipi_recv(); +#endif } #endif