Index: kernel/arch/abs32le/include/mm/page.h =================================================================== --- kernel/arch/abs32le/include/mm/page.h (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/arch/abs32le/include/mm/page.h (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -105,4 +105,12 @@ set_pt_flags((pte_t *) (ptl3), (size_t) (i), (x)) +/* Set PTE present bit accessors for each level. */ +#define SET_PTL1_PRESENT_ARCH(ptl0, i) \ + set_pt_present((pte_t *) (ptl0), (size_t) (i)) +#define SET_PTL2_PRESENT_ARCH(ptl1, i) +#define SET_PTL3_PRESENT_ARCH(ptl2, i) +#define SET_FRAME_PRESENT_ARCH(ptl3, i) \ + set_pt_present((pte_t *) (ptl3), (size_t) (i)) + /* Macros for querying the last level entries. */ #define PTE_VALID_ARCH(p) \ @@ -173,4 +181,13 @@ } +NO_TRACE static inline void set_pt_present(pte_t *pt, size_t i) + WRITES(ARRAY_RANGE(pt, PTL0_ENTRIES_ARCH)) + REQUIRES_ARRAY_MUTABLE(pt, PTL0_ENTRIES_ARCH) +{ + pte_t *p = &pt[i]; + + p->present = 1; +} + extern void page_arch_init(void); extern void page_fault(unsigned int, istate_t *); Index: kernel/arch/amd64/include/mm/page.h =================================================================== --- kernel/arch/amd64/include/mm/page.h (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/arch/amd64/include/mm/page.h (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -119,4 +119,14 @@ set_pt_flags((pte_t *) (ptl3), (size_t) (i), (x)) +/* Set PTE present bit accessors for each level. */ +#define SET_PTL1_PRESENT_ARCH(ptl0, i) \ + set_pt_present((pte_t *) (ptl0), (size_t) (i)) +#define SET_PTL2_PRESENT_ARCH(ptl1, i) \ + set_pt_present((pte_t *) (ptl1), (size_t) (i)) +#define SET_PTL3_PRESENT_ARCH(ptl2, i) \ + set_pt_present((pte_t *) (ptl2), (size_t) (i)) +#define SET_FRAME_PRESENT_ARCH(ptl3, i) \ + set_pt_present((pte_t *) (ptl3), (size_t) (i)) + /* Macros for querying the last-level PTE entries. */ #define PTE_VALID_ARCH(p) \ @@ -215,4 +225,11 @@ } +NO_TRACE static inline void set_pt_present(pte_t *pt, size_t i) +{ + pte_t *p = &pt[i]; + + p->present = 1; +} + extern void page_arch_init(void); extern void page_fault(unsigned int, istate_t *); Index: kernel/arch/amd64/src/mm/page.c =================================================================== --- kernel/arch/amd64/src/mm/page.c (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/arch/amd64/src/mm/page.c (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -57,5 +57,5 @@ uintptr_t cur; unsigned int identity_flags = - PAGE_CACHEABLE | PAGE_EXEC | PAGE_GLOBAL | PAGE_WRITE; + PAGE_GLOBAL | PAGE_CACHEABLE | PAGE_EXEC | PAGE_WRITE | PAGE_READ; page_mapping_operations = &pt_mapping_operations; Index: kernel/arch/arm32/include/mm/page.h =================================================================== --- kernel/arch/arm32/include/mm/page.h (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/arch/arm32/include/mm/page.h (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -40,4 +40,5 @@ #include #include +#include #include @@ -109,4 +110,12 @@ #define SET_FRAME_FLAGS_ARCH(ptl3, i, x) \ set_pt_level1_flags((pte_t *) (ptl3), (size_t) (i), (x)) + +/* Set PTE present bit accessors for each level. */ +#define SET_PTL1_PRESENT_ARCH(ptl0, i) \ + set_pt_level0_present((pte_t *) (ptl0), (size_t) (i)) +#define SET_PTL2_PRESENT_ARCH(ptl1, i) +#define SET_PTL3_PRESENT_ARCH(ptl2, i) +#define SET_FRAME_PRESENT_ARCH(ptl3, i) \ + set_pt_level1_present((pte_t *) (ptl3), (size_t) (i)) /* Macros for querying the last-level PTE entries. */ @@ -267,4 +276,12 @@ } +NO_TRACE static inline void set_pt_level0_present(pte_t *pt, size_t i) +{ + pte_level0_t *p = &pt[i].l0; + + p->should_be_zero = 0; + write_barrier(); + p->descriptor_type = PTE_DESCRIPTOR_COARSE_TABLE; +} /** Sets flags of level 1 page table entry. @@ -283,11 +300,8 @@ pte_level1_t *p = &pt[i].l1; - if (flags & PAGE_NOT_PRESENT) { + if (flags & PAGE_NOT_PRESENT) p->descriptor_type = PTE_DESCRIPTOR_NOT_PRESENT; - p->access_permission_3 = 1; - } else { + else p->descriptor_type = PTE_DESCRIPTOR_SMALL_PAGE; - p->access_permission_3 = p->access_permission_0; - } p->cacheable = p->bufferable = (flags & PAGE_CACHEABLE) != 0; @@ -312,8 +326,13 @@ } - +NO_TRACE static inline void set_pt_level1_present(pte_t *pt, size_t i) +{ + pte_level1_t *p = &pt[i].l1; + + p->descriptor_type = PTE_DESCRIPTOR_SMALL_PAGE; +} + extern void page_arch_init(void); - #endif /* __ASM__ */ Index: kernel/arch/ia32/include/mm/page.h =================================================================== --- kernel/arch/ia32/include/mm/page.h (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/arch/ia32/include/mm/page.h (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -115,4 +115,12 @@ set_pt_flags((pte_t *) (ptl3), (size_t) (i), (x)) +/* Set PTE present bit accessors for each level. */ +#define SET_PTL1_PRESENT_ARCH(ptl0, i) \ + set_pt_present((pte_t *) (ptl0), (size_t) (i)) +#define SET_PTL2_PRESENT_ARCH(ptl1, i) +#define SET_PTL3_PRESENT_ARCH(ptl2, i) +#define SET_FRAME_PRESENT_ARCH(ptl3, i) \ + set_pt_present((pte_t *) (ptl3), (size_t) (i)) + /* Macros for querying the last level entries. */ #define PTE_VALID_ARCH(p) \ @@ -194,4 +202,11 @@ } +NO_TRACE static inline void set_pt_present(pte_t *pt, size_t i) +{ + pte_t *p = &pt[i]; + + p->present = 1; +} + extern void page_arch_init(void); extern void page_fault(unsigned int, istate_t *); Index: kernel/arch/ia32/src/mm/page.c =================================================================== --- kernel/arch/ia32/src/mm/page.c (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/arch/ia32/src/mm/page.c (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -71,8 +71,5 @@ for (cur = 0; cur < min(config.identity_size, config.physmem_end); cur += FRAME_SIZE) { - flags = PAGE_CACHEABLE | PAGE_WRITE; - if ((PA2KA(cur) >= config.base) && - (PA2KA(cur) < config.base + config.kernel_size)) - flags |= PAGE_GLOBAL; + flags = PAGE_GLOBAL | PAGE_CACHEABLE | PAGE_WRITE | PAGE_READ; page_mapping_insert(AS_KERNEL, PA2KA(cur), cur, flags); } Index: kernel/arch/ia32/src/smp/apic.c =================================================================== --- kernel/arch/ia32/src/smp/apic.c (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/arch/ia32/src/smp/apic.c (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -259,4 +259,24 @@ } +#define DELIVS_PENDING_SILENT_RETRIES 4 + +static void l_apic_wait_for_delivery(void) +{ + icr_t icr; + unsigned retries = 0; + + do { + if (retries++ > DELIVS_PENDING_SILENT_RETRIES) { + retries = 0; +#ifdef CONFIG_DEBUG + printf("IPI is pending.\n"); +#endif + delay(20); + } + icr.lo = l_apic[ICRlo]; + } while (icr.delivs == DELIVS_PENDING); + +} + /** Send all CPUs excluding CPU IPI vector. * @@ -279,11 +299,6 @@ l_apic[ICRlo] = icr.lo; - - icr.lo = l_apic[ICRlo]; - if (icr.delivs == DELIVS_PENDING) { -#ifdef CONFIG_DEBUG - printf("IPI is pending.\n"); -#endif - } + + l_apic_wait_for_delivery(); return apic_poll_errors(); @@ -327,11 +342,7 @@ return 0; + l_apic_wait_for_delivery(); + icr.lo = l_apic[ICRlo]; - if (icr.delivs == DELIVS_PENDING) { -#ifdef CONFIG_DEBUG - printf("IPI is pending.\n"); -#endif - } - icr.delmod = DELMOD_INIT; icr.destmod = DESTMOD_PHYS; Index: kernel/arch/mips32/include/mm/page.h =================================================================== --- kernel/arch/mips32/include/mm/page.h (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/arch/mips32/include/mm/page.h (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -128,4 +128,12 @@ set_pt_flags((pte_t *) (ptl3), (size_t) (i), (x)) +/* Set PTE present bit accessors for each level. */ +#define SET_PTL1_PRESENT_ARCH(ptl0, i) \ + set_pt_present((pte_t *) (ptl0), (size_t) (i)) +#define SET_PTL2_PRESENT_ARCH(ptl1, i) +#define SET_PTL3_PRESENT_ARCH(ptl2, i) +#define SET_FRAME_PRESENT_ARCH(ptl3, i) \ + set_pt_present((pte_t *) (ptl3), (size_t) (i)) + /* Last-level info macros. */ #define PTE_VALID_ARCH(pte) (*((uint32_t *) (pte)) != 0) @@ -182,4 +190,12 @@ } +NO_TRACE static inline void set_pt_present(pte_t *pt, size_t i) +{ + pte_t *p = &pt[i]; + + p->p = 1; +} + + extern void page_arch_init(void); Index: kernel/arch/ppc32/include/mm/page.h =================================================================== --- kernel/arch/ppc32/include/mm/page.h (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/arch/ppc32/include/mm/page.h (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -128,4 +128,14 @@ #define SET_FRAME_FLAGS_ARCH(ptl3, i, x) \ set_pt_flags((pte_t *) (ptl3), (size_t) (i), (x)) + +/* Set PTE present accessors for each level. */ +#define SET_PTL1_PRESENT_ARCH(ptl0, i) \ + set_pt_present((pte_t *) (ptl0), (size_t) (i)) + +#define SET_PTL2_PRESENT_ARCH(ptl1, i) +#define SET_PTL3_PRESENT_ARCH(ptl2, i) + +#define SET_FRAME_PRESENT_ARCH(ptl3, i) \ + set_pt_present((pte_t *) (ptl3), (size_t) (i)) /* Macros for querying the last-level PTEs. */ @@ -175,4 +185,11 @@ } +NO_TRACE static inline void set_pt_present(pte_t *pt, size_t i) +{ + pte_t *entry = &pt[i]; + + entry->present = 1; +} + extern void page_arch_init(void); Index: kernel/arch/sparc64/src/smp/sun4u/ipi.c =================================================================== --- kernel/arch/sparc64/src/smp/sun4u/ipi.c (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/arch/sparc64/src/smp/sun4u/ipi.c (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -124,5 +124,5 @@ (void) interrupts_disable(); } - } while (done); + } while (!done); preemption_enable(); Index: kernel/genarch/include/mm/page_pt.h =================================================================== --- kernel/genarch/include/mm/page_pt.h (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/genarch/include/mm/page_pt.h (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -115,4 +115,13 @@ /* + * These macros are provided to set the present bit within the page tables. + * + */ +#define SET_PTL1_PRESENT(ptl0, i) SET_PTL1_PRESENT_ARCH(ptl0, i) +#define SET_PTL2_PRESENT(ptl1, i) SET_PTL2_PRESENT_ARCH(ptl1, i) +#define SET_PTL3_PRESENT(ptl2, i) SET_PTL3_PRESENT_ARCH(ptl2, i) +#define SET_FRAME_PRESENT(ptl3, i) SET_FRAME_PRESENT_ARCH(ptl3, i) + +/* * Macros for querying the last-level PTEs. * Index: kernel/genarch/src/mm/page_ht.c =================================================================== --- kernel/genarch/src/mm/page_ht.c (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/genarch/src/mm/page_ht.c (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -45,4 +45,5 @@ #include #include +#include #include #include @@ -207,4 +208,6 @@ pte->page = ALIGN_DOWN(page, PAGE_SIZE); pte->frame = ALIGN_DOWN(frame, FRAME_SIZE); + + write_barrier(); hash_table_insert(&page_ht, key, &pte->link); Index: kernel/genarch/src/mm/page_pt.c =================================================================== --- kernel/genarch/src/mm/page_pt.c (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/genarch/src/mm/page_pt.c (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -43,4 +43,5 @@ #include #include +#include #include #include @@ -86,6 +87,8 @@ SET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page), KA2PA(newpt)); SET_PTL1_FLAGS(ptl0, PTL0_INDEX(page), - PAGE_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE | + PAGE_NOT_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE | PAGE_WRITE); + write_barrier(); + SET_PTL1_PRESENT(ptl0, PTL0_INDEX(page)); } @@ -98,6 +101,8 @@ SET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page), KA2PA(newpt)); SET_PTL2_FLAGS(ptl1, PTL1_INDEX(page), - PAGE_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE | + PAGE_NOT_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE | PAGE_WRITE); + write_barrier(); + SET_PTL2_PRESENT(ptl1, PTL1_INDEX(page)); } @@ -110,6 +115,8 @@ SET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page), KA2PA(newpt)); SET_PTL3_FLAGS(ptl2, PTL2_INDEX(page), - PAGE_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE | + PAGE_NOT_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE | PAGE_WRITE); + write_barrier(); + SET_PTL3_PRESENT(ptl2, PTL2_INDEX(page)); } @@ -117,5 +124,7 @@ SET_FRAME_ADDRESS(ptl3, PTL3_INDEX(page), frame); - SET_FRAME_FLAGS(ptl3, PTL3_INDEX(page), flags); + SET_FRAME_FLAGS(ptl3, PTL3_INDEX(page), flags | PAGE_NOT_PRESENT); + write_barrier(); + SET_FRAME_PRESENT(ptl3, PTL3_INDEX(page)); } @@ -279,12 +288,22 @@ if (GET_PTL1_FLAGS(ptl0, PTL0_INDEX(page)) & PAGE_NOT_PRESENT) return NULL; + + read_barrier(); pte_t *ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page))); if (GET_PTL2_FLAGS(ptl1, PTL1_INDEX(page)) & PAGE_NOT_PRESENT) return NULL; + +#if (PTL1_ENTRIES != 0) + read_barrier(); +#endif pte_t *ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page))); if (GET_PTL3_FLAGS(ptl2, PTL2_INDEX(page)) & PAGE_NOT_PRESENT) return NULL; + +#if (PTL2_ENTRIES != 0) + read_barrier(); +#endif pte_t *ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page))); @@ -346,6 +365,6 @@ SET_PTL1_ADDRESS(ptl0, PTL0_INDEX(addr), KA2PA(l1)); SET_PTL1_FLAGS(ptl0, PTL0_INDEX(addr), - PAGE_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE | - PAGE_WRITE); + PAGE_PRESENT | PAGE_USER | PAGE_CACHEABLE | + PAGE_EXEC | PAGE_WRITE | PAGE_READ); } } Index: kernel/generic/src/mm/as.c =================================================================== --- kernel/generic/src/mm/as.c (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/generic/src/mm/as.c (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -665,10 +665,4 @@ page_table_lock(as, false); - - /* - * Start TLB shootdown sequence. - */ - ipl_t ipl = tlb_shootdown_start(TLB_INVL_PAGES, as->asid, - area->base + P2SZ(pages), area->pages - pages); /* @@ -726,4 +720,20 @@ } + /* + * Start TLB shootdown sequence. + * + * The sequence is rather short and can be + * repeated multiple times. The reason is that + * we don't want to have used_space_remove() + * inside the sequence as it may use a blocking + * memory allocation for its B+tree. Blocking + * while holding the tlblock spinlock is + * forbidden and would hit a kernel assertion. + */ + + ipl_t ipl = tlb_shootdown_start(TLB_INVL_PAGES, + as->asid, area->base + P2SZ(pages), + area->pages - pages); + for (; i < size; i++) { pte_t *pte = page_mapping_find(as, @@ -743,22 +753,23 @@ page_mapping_remove(as, ptr + P2SZ(i)); } + + /* + * Finish TLB shootdown sequence. + */ + + tlb_invalidate_pages(as->asid, + area->base + P2SZ(pages), + area->pages - pages); + + /* + * Invalidate software translation caches + * (e.g. TSB on sparc64, PHT on ppc32). + */ + as_invalidate_translation_cache(as, + area->base + P2SZ(pages), + area->pages - pages); + tlb_shootdown_finalize(ipl); } } - - /* - * Finish TLB shootdown sequence. - */ - - tlb_invalidate_pages(as->asid, area->base + P2SZ(pages), - area->pages - pages); - - /* - * Invalidate software translation caches - * (e.g. TSB on sparc64, PHT on ppc32). - */ - as_invalidate_translation_cache(as, area->base + P2SZ(pages), - area->pages - pages); - tlb_shootdown_finalize(ipl); - page_table_unlock(as, false); } else { Index: kernel/generic/src/mm/tlb.c =================================================================== --- kernel/generic/src/mm/tlb.c (revision 96606fe85ef15212d997e4ef8df25a495215a6c9) +++ kernel/generic/src/mm/tlb.c (revision cc51c27327e544ae766a074d3153e36db67aef25) @@ -162,5 +162,5 @@ size_t i; - for (i = 0; i < CPU->tlb_messages_count; CPU->tlb_messages_count--) { + for (i = 0; i < CPU->tlb_messages_count; i++) { tlb_invalidate_type_t type = CPU->tlb_messages[i].type; asid_t asid = CPU->tlb_messages[i].asid; @@ -188,4 +188,5 @@ } + CPU->tlb_messages_count = 0; irq_spinlock_unlock(&CPU->lock, false); CPU->tlb_active = true;