source: mainline/kernel/genarch/src/mm/page_pt.c@ 609a417

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since 609a417 was 609a417, checked in by Jakub Jermar <jakub@…>, 13 years ago

Add write memory barriers to pt_mapping_insert() so that setting a new
page table level is separated from marking it present in the previous
level.

This is to make sure that a concurrent hardware page table walk or
pt_page_find() will see the new level only after it is fully
initialized.

  • Property mode set to 100644
File size: 10.4 KB
RevLine 
[6d7ffa65]1/*
[df4ed85]2 * Copyright (c) 2006 Jakub Jermar
[6d7ffa65]3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
[f47fd19]29/** @addtogroup genarchmm
[b45c443]30 * @{
31 */
32
[0f27b4c]33/**
[b45c443]34 * @file
[da1bafb]35 * @brief Virtual Address Translation for hierarchical 4-level page tables.
[0f27b4c]36 */
37
[6d7ffa65]38#include <genarch/mm/page_pt.h>
39#include <mm/page.h>
40#include <mm/frame.h>
[c72dc15]41#include <mm/km.h>
[ef67bab]42#include <mm/as.h>
[6d7ffa65]43#include <arch/mm/page.h>
[fc1e4f6]44#include <arch/mm/as.h>
[609a417]45#include <arch/barrier.h>
[d99c1d2]46#include <typedefs.h>
[6d7ffa65]47#include <arch/asm.h>
48#include <memstr.h>
[c868e2d]49#include <align.h>
50#include <macros.h>
[caed0279]51#include <bitops.h>
[6d7ffa65]52
[da1bafb]53static void pt_mapping_insert(as_t *, uintptr_t, uintptr_t, unsigned int);
54static void pt_mapping_remove(as_t *, uintptr_t);
[235e6c7]55static pte_t *pt_mapping_find(as_t *, uintptr_t, bool);
[c868e2d]56static void pt_mapping_make_global(uintptr_t, size_t);
[6d7ffa65]57
[f5935ed]58page_mapping_operations_t pt_mapping_operations = {
[6d7ffa65]59 .mapping_insert = pt_mapping_insert,
[8f00329]60 .mapping_remove = pt_mapping_remove,
[c868e2d]61 .mapping_find = pt_mapping_find,
62 .mapping_make_global = pt_mapping_make_global
[6d7ffa65]63};
64
65/** Map page to frame using hierarchical page tables.
66 *
[9179d0a]67 * Map virtual address page to physical address frame
68 * using flags.
[6d7ffa65]69 *
[da1bafb]70 * @param as Address space to wich page belongs.
71 * @param page Virtual address of the page to be mapped.
[6d7ffa65]72 * @param frame Physical address of memory frame to which the mapping is done.
73 * @param flags Flags to be used for mapping.
[da1bafb]74 *
[6d7ffa65]75 */
[da1bafb]76void pt_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame,
77 unsigned int flags)
[6d7ffa65]78{
[da1bafb]79 pte_t *ptl0 = (pte_t *) PA2KA((uintptr_t) as->genarch.page_table);
[1d432f9]80
81 ASSERT(page_table_locked(as));
[da1bafb]82
[6d7ffa65]83 if (GET_PTL1_FLAGS(ptl0, PTL0_INDEX(page)) & PAGE_NOT_PRESENT) {
[6b326ea1]84 pte_t *newpt = (pte_t *) frame_alloc(PTL1_SIZE,
85 FRAME_LOWMEM | FRAME_KA);
[e32e092]86 memsetb(newpt, FRAME_SIZE << PTL1_SIZE, 0);
[6d7ffa65]87 SET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page), KA2PA(newpt));
[6b326ea1]88 SET_PTL1_FLAGS(ptl0, PTL0_INDEX(page),
[609a417]89 PAGE_NOT_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
[6b326ea1]90 PAGE_WRITE);
[609a417]91 write_barrier();
92 SET_PTL1_PRESENT(ptl0, PTL0_INDEX(page));
[6d7ffa65]93 }
[da1bafb]94
95 pte_t *ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page)));
96
[6d7ffa65]97 if (GET_PTL2_FLAGS(ptl1, PTL1_INDEX(page)) & PAGE_NOT_PRESENT) {
[6b326ea1]98 pte_t *newpt = (pte_t *) frame_alloc(PTL2_SIZE,
99 FRAME_LOWMEM | FRAME_KA);
[e32e092]100 memsetb(newpt, FRAME_SIZE << PTL2_SIZE, 0);
[6d7ffa65]101 SET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page), KA2PA(newpt));
[6b326ea1]102 SET_PTL2_FLAGS(ptl1, PTL1_INDEX(page),
[609a417]103 PAGE_NOT_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
[6b326ea1]104 PAGE_WRITE);
[609a417]105 write_barrier();
106 SET_PTL2_PRESENT(ptl1, PTL1_INDEX(page));
[6d7ffa65]107 }
[da1bafb]108
109 pte_t *ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page)));
110
[6d7ffa65]111 if (GET_PTL3_FLAGS(ptl2, PTL2_INDEX(page)) & PAGE_NOT_PRESENT) {
[6b326ea1]112 pte_t *newpt = (pte_t *) frame_alloc(PTL3_SIZE,
113 FRAME_LOWMEM | FRAME_KA);
[e32e092]114 memsetb(newpt, FRAME_SIZE << PTL3_SIZE, 0);
[6d7ffa65]115 SET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page), KA2PA(newpt));
[6b326ea1]116 SET_PTL3_FLAGS(ptl2, PTL2_INDEX(page),
[609a417]117 PAGE_NOT_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
[6b326ea1]118 PAGE_WRITE);
[609a417]119 write_barrier();
120 SET_PTL3_PRESENT(ptl2, PTL2_INDEX(page));
[6d7ffa65]121 }
[da1bafb]122
123 pte_t *ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page)));
124
[6d7ffa65]125 SET_FRAME_ADDRESS(ptl3, PTL3_INDEX(page), frame);
[609a417]126 SET_FRAME_FLAGS(ptl3, PTL3_INDEX(page), flags | PAGE_NOT_PRESENT);
127 write_barrier();
128 SET_FRAME_PRESENT(ptl3, PTL3_INDEX(page));
[6d7ffa65]129}
130
[8f00329]131/** Remove mapping of page from hierarchical page tables.
132 *
[9179d0a]133 * Remove any mapping of page within address space as.
[8f00329]134 * TLB shootdown should follow in order to make effects of
135 * this call visible.
136 *
[ecbdc724]137 * Empty page tables except PTL0 are freed.
138 *
[da1bafb]139 * @param as Address space to wich page belongs.
[8f00329]140 * @param page Virtual address of the page to be demapped.
[da1bafb]141 *
[8f00329]142 */
[7f1c620]143void pt_mapping_remove(as_t *as, uintptr_t page)
[8f00329]144{
[1d432f9]145 ASSERT(page_table_locked(as));
146
[ecbdc724]147 /*
148 * First, remove the mapping, if it exists.
149 */
[da1bafb]150
151 pte_t *ptl0 = (pte_t *) PA2KA((uintptr_t) as->genarch.page_table);
[8f00329]152 if (GET_PTL1_FLAGS(ptl0, PTL0_INDEX(page)) & PAGE_NOT_PRESENT)
153 return;
[da1bafb]154
155 pte_t *ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page)));
[8f00329]156 if (GET_PTL2_FLAGS(ptl1, PTL1_INDEX(page)) & PAGE_NOT_PRESENT)
157 return;
[da1bafb]158
159 pte_t *ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page)));
[8f00329]160 if (GET_PTL3_FLAGS(ptl2, PTL2_INDEX(page)) & PAGE_NOT_PRESENT)
161 return;
[da1bafb]162
163 pte_t *ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page)));
164
[c868e2d]165 /*
166 * Destroy the mapping.
167 * Setting to PAGE_NOT_PRESENT is not sufficient.
168 */
[e32e092]169 memsetb(&ptl3[PTL3_INDEX(page)], sizeof(pte_t), 0);
[da1bafb]170
[ecbdc724]171 /*
[c72dc15]172 * Second, free all empty tables along the way from PTL3 down to PTL0
173 * except those needed for sharing the kernel non-identity mappings.
[ecbdc724]174 */
175
[da1bafb]176 /* Check PTL3 */
177 bool empty = true;
178
179 unsigned int i;
[ecbdc724]180 for (i = 0; i < PTL3_ENTRIES; i++) {
181 if (PTE_VALID(&ptl3[i])) {
182 empty = false;
183 break;
184 }
185 }
[da1bafb]186
[ecbdc724]187 if (empty) {
188 /*
189 * PTL3 is empty.
[c72dc15]190 * Release the frame and remove PTL3 pointer from the parent
191 * table.
[ecbdc724]192 */
[da1bafb]193#if (PTL2_ENTRIES != 0)
194 memsetb(&ptl2[PTL2_INDEX(page)], sizeof(pte_t), 0);
195#elif (PTL1_ENTRIES != 0)
196 memsetb(&ptl1[PTL1_INDEX(page)], sizeof(pte_t), 0);
197#else
[c72dc15]198 if (km_is_non_identity(page))
199 return;
200
[da1bafb]201 memsetb(&ptl0[PTL0_INDEX(page)], sizeof(pte_t), 0);
202#endif
[c72dc15]203 frame_free(KA2PA((uintptr_t) ptl3));
[ecbdc724]204 } else {
205 /*
206 * PTL3 is not empty.
207 * Therefore, there must be a path from PTL0 to PTL3 and
208 * thus nothing to free in higher levels.
[da1bafb]209 *
[ecbdc724]210 */
211 return;
212 }
213
[da1bafb]214 /* Check PTL2, empty is still true */
215#if (PTL2_ENTRIES != 0)
216 for (i = 0; i < PTL2_ENTRIES; i++) {
217 if (PTE_VALID(&ptl2[i])) {
218 empty = false;
219 break;
[ecbdc724]220 }
221 }
[da1bafb]222
223 if (empty) {
224 /*
225 * PTL2 is empty.
[c72dc15]226 * Release the frame and remove PTL2 pointer from the parent
227 * table.
[da1bafb]228 */
229#if (PTL1_ENTRIES != 0)
230 memsetb(&ptl1[PTL1_INDEX(page)], sizeof(pte_t), 0);
231#else
[c72dc15]232 if (km_is_non_identity(page))
233 return;
234
[da1bafb]235 memsetb(&ptl0[PTL0_INDEX(page)], sizeof(pte_t), 0);
236#endif
[c72dc15]237 frame_free(KA2PA((uintptr_t) ptl2));
[da1bafb]238 } else {
239 /*
240 * PTL2 is not empty.
241 * Therefore, there must be a path from PTL0 to PTL2 and
242 * thus nothing to free in higher levels.
243 *
244 */
245 return;
246 }
247#endif /* PTL2_ENTRIES != 0 */
248
[ecbdc724]249 /* check PTL1, empty is still true */
[da1bafb]250#if (PTL1_ENTRIES != 0)
251 for (i = 0; i < PTL1_ENTRIES; i++) {
252 if (PTE_VALID(&ptl1[i])) {
253 empty = false;
254 break;
[ecbdc724]255 }
256 }
[da1bafb]257
258 if (empty) {
259 /*
260 * PTL1 is empty.
[c72dc15]261 * Release the frame and remove PTL1 pointer from the parent
262 * table.
[da1bafb]263 */
[c72dc15]264 if (km_is_non_identity(page))
265 return;
266
[da1bafb]267 memsetb(&ptl0[PTL0_INDEX(page)], sizeof(pte_t), 0);
[c72dc15]268 frame_free(KA2PA((uintptr_t) ptl1));
[da1bafb]269 }
270#endif /* PTL1_ENTRIES != 0 */
[8f00329]271}
272
[6d7ffa65]273/** Find mapping for virtual page in hierarchical page tables.
274 *
[235e6c7]275 * @param as Address space to which page belongs.
276 * @param page Virtual page.
277 * @param nolock True if the page tables need not be locked.
[6d7ffa65]278 *
[da1bafb]279 * @return NULL if there is no such mapping; entry from PTL3 describing
280 * the mapping otherwise.
281 *
[6d7ffa65]282 */
[235e6c7]283pte_t *pt_mapping_find(as_t *as, uintptr_t page, bool nolock)
[6d7ffa65]284{
[235e6c7]285 ASSERT(nolock || page_table_locked(as));
[1d432f9]286
[da1bafb]287 pte_t *ptl0 = (pte_t *) PA2KA((uintptr_t) as->genarch.page_table);
[6d7ffa65]288 if (GET_PTL1_FLAGS(ptl0, PTL0_INDEX(page)) & PAGE_NOT_PRESENT)
289 return NULL;
[da1bafb]290
291 pte_t *ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page)));
[6d7ffa65]292 if (GET_PTL2_FLAGS(ptl1, PTL1_INDEX(page)) & PAGE_NOT_PRESENT)
293 return NULL;
[da1bafb]294
295 pte_t *ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page)));
[6d7ffa65]296 if (GET_PTL3_FLAGS(ptl2, PTL2_INDEX(page)) & PAGE_NOT_PRESENT)
297 return NULL;
[da1bafb]298
299 pte_t *ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page)));
300
[6d7ffa65]301 return &ptl3[PTL3_INDEX(page)];
302}
[b45c443]303
[caed0279]304/** Return the size of the region mapped by a single PTL0 entry.
305 *
306 * @return Size of the region mapped by a single PTL0 entry.
307 */
308static uintptr_t ptl0_step_get(void)
309{
310 size_t va_bits;
311
312 va_bits = fnzb(PTL0_ENTRIES) + fnzb(PTL1_ENTRIES) + fnzb(PTL2_ENTRIES) +
313 fnzb(PTL3_ENTRIES) + PAGE_WIDTH;
314
315 return 1UL << (va_bits - fnzb(PTL0_ENTRIES));
316}
317
[c868e2d]318/** Make the mappings in the given range global accross all address spaces.
319 *
320 * All PTL0 entries in the given range will be mapped to a next level page
321 * table. The next level page table will be allocated and cleared.
322 *
323 * pt_mapping_remove() will never deallocate these page tables even when there
324 * are no PTEs in them.
325 *
326 * @param as Address space.
327 * @param base Base address corresponding to the first PTL0 entry that will be
328 * altered by this function.
329 * @param size Size in bytes defining the range of PTL0 entries that will be
330 * altered by this function.
331 */
332void pt_mapping_make_global(uintptr_t base, size_t size)
333{
334 uintptr_t ptl0 = PA2KA((uintptr_t) AS_KERNEL->genarch.page_table);
[caed0279]335 uintptr_t ptl0_step = ptl0_step_get();
[c868e2d]336 size_t order;
337 uintptr_t addr;
338
339#if (PTL1_ENTRIES != 0)
340 order = PTL1_SIZE;
341#elif (PTL2_ENTRIES != 0)
342 order = PTL2_SIZE;
343#else
344 order = PTL3_SIZE;
345#endif
346
[a2789d2]347 ASSERT(size > 0);
[c868e2d]348
[caed0279]349 for (addr = ALIGN_DOWN(base, ptl0_step); addr - 1 < base + size - 1;
350 addr += ptl0_step) {
[c868e2d]351 uintptr_t l1;
352
353 l1 = (uintptr_t) frame_alloc(order, FRAME_KA | FRAME_LOWMEM);
354 memsetb((void *) l1, FRAME_SIZE << order, 0);
355 SET_PTL1_ADDRESS(ptl0, PTL0_INDEX(addr), KA2PA(l1));
356 SET_PTL1_FLAGS(ptl0, PTL0_INDEX(addr),
[34ab31c0]357 PAGE_PRESENT | PAGE_USER | PAGE_CACHEABLE |
358 PAGE_EXEC | PAGE_WRITE | PAGE_READ);
[c868e2d]359 }
360}
361
[f47fd19]362/** @}
[b45c443]363 */
Note: See TracBrowser for help on using the repository browser.