source: mainline/kernel/arch/ppc32/src/mm/tlb.c@ 4872160

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since 4872160 was a000878c, checked in by Martin Decky <martin@…>, 15 years ago

make sure that all statically allocated strings are declared as "const char *"
and are treated as read-only
  • Property mode set to 100644
File size: 11.5 KB
RevLine 
[613bc54]1/*
[df4ed85]2 * Copyright (c) 2006 Martin Decky
[613bc54]3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
[7b187ef]29/** @addtogroup ppc32mm
[b45c443]30 * @{
31 */
32/** @file
33 */
34
[613bc54]35#include <mm/tlb.h>
[10e0cee]36#include <arch/mm/tlb.h>
37#include <arch/interrupt.h>
[26fa0f9f]38#include <interrupt.h>
[10e0cee]39#include <mm/as.h>
[eae4e8f]40#include <mm/page.h>
[10e0cee]41#include <arch.h>
42#include <print.h>
[53634f9]43#include <macros.h>
[e2b762ec]44#include <symtab.h>
[9a68b34d]45
[896ad9f]46static unsigned int seed = 10;
47static unsigned int seed_real __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42;
48
49
[10e0cee]50/** Try to find PTE for faulting address
[613bc54]51 *
[10e0cee]52 * Try to find PTE for faulting address.
53 * The as->lock must be held on entry to this function
54 * if lock is true.
55 *
[5d67baa]56 * @param as Address space.
57 * @param lock Lock/unlock the address space.
58 * @param badvaddr Faulting virtual address.
59 * @param access Access mode that caused the fault.
60 * @param istate Pointer to interrupted state.
61 * @param pfrc Pointer to variable where as_page_fault() return code
62 * will be stored.
63 * @return PTE on success, NULL otherwise.
[613bc54]64 *
65 */
[5d67baa]66static pte_t *
67find_mapping_and_check(as_t *as, bool lock, uintptr_t badvaddr, int access,
68 istate_t *istate, int *pfrc)
[10e0cee]69{
70 /*
71 * Check if the mapping exists in page tables.
72 */
73 pte_t *pte = page_mapping_find(as, badvaddr);
[43d6401]74 if ((pte) && (pte->present)) {
[10e0cee]75 /*
76 * Mapping found in page tables.
77 * Immediately succeed.
78 */
79 return pte;
80 } else {
81 int rc;
82
83 /*
84 * Mapping not found in page tables.
85 * Resort to higher-level page fault handler.
86 */
87 page_table_unlock(as, lock);
88 switch (rc = as_page_fault(badvaddr, access, istate)) {
[5d67baa]89 case AS_PF_OK:
90 /*
91 * The higher-level page fault handler succeeded,
92 * The mapping ought to be in place.
93 */
94 page_table_lock(as, lock);
95 pte = page_mapping_find(as, badvaddr);
[43d6401]96 ASSERT((pte) && (pte->present));
[5d67baa]97 *pfrc = 0;
98 return pte;
99 case AS_PF_DEFER:
100 page_table_lock(as, lock);
101 *pfrc = rc;
102 return NULL;
103 case AS_PF_FAULT:
[37e518b]104 page_table_lock(as, lock);
[5d67baa]105 *pfrc = rc;
106 return NULL;
107 default:
[f651e80]108 panic("Unexpected rc (%d).", rc);
[10e0cee]109 }
110 }
111}
112
113
[7f1c620]114static void pht_refill_fail(uintptr_t badvaddr, istate_t *istate)
[10e0cee]115{
[a000878c]116 const char *symbol = symtab_fmt_name_lookup(istate->pc);
117 const char *sym2 = symtab_fmt_name_lookup(istate->lr);
118
[26fa0f9f]119 fault_if_from_uspace(istate,
[826c203]120 "PHT Refill Exception on %p.", badvaddr);
[f651e80]121 panic("%p: PHT Refill Exception at %p (%s<-%s).", badvaddr,
[5d67baa]122 istate->pc, symbol, sym2);
[10e0cee]123}
124
125
[43d6401]126static void pht_insert(const uintptr_t vaddr, const pte_t *pte)
[10e0cee]127{
[7f1c620]128 uint32_t page = (vaddr >> 12) & 0xffff;
129 uint32_t api = (vaddr >> 22) & 0x3f;
[10e0cee]130
[7f1c620]131 uint32_t vsid;
[10e0cee]132 asm volatile (
133 "mfsrin %0, %1\n"
134 : "=r" (vsid)
135 : "r" (vaddr)
136 );
137
[7f1c620]138 uint32_t sdr1;
[10e0cee]139 asm volatile (
140 "mfsdr1 %0\n"
141 : "=r" (sdr1)
142 );
143 phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
144
145 /* Primary hash (xor) */
[7f1c620]146 uint32_t h = 0;
147 uint32_t hash = vsid ^ page;
148 uint32_t base = (hash & 0x3ff) << 3;
149 uint32_t i;
[10e0cee]150 bool found = false;
151
[1e241723]152 /* Find colliding PTE in PTEG */
[10e0cee]153 for (i = 0; i < 8; i++) {
[1e241723]154 if ((phte[base + i].v)
155 && (phte[base + i].vsid == vsid)
[7c98874]156 && (phte[base + i].api == api)
[1e241723]157 && (phte[base + i].h == 0)) {
[10e0cee]158 found = true;
159 break;
160 }
161 }
162
[1e241723]163 if (!found) {
164 /* Find unused PTE in PTEG */
165 for (i = 0; i < 8; i++) {
166 if (!phte[base + i].v) {
167 found = true;
168 break;
169 }
170 }
171 }
172
[10e0cee]173 if (!found) {
174 /* Secondary hash (not) */
[7f1c620]175 uint32_t base2 = (~hash & 0x3ff) << 3;
[10e0cee]176
[1e241723]177 /* Find colliding PTE in PTEG */
[10e0cee]178 for (i = 0; i < 8; i++) {
[1e241723]179 if ((phte[base2 + i].v)
180 && (phte[base2 + i].vsid == vsid)
[7c98874]181 && (phte[base2 + i].api == api)
[1e241723]182 && (phte[base2 + i].h == 1)) {
[10e0cee]183 found = true;
184 base = base2;
185 h = 1;
186 break;
187 }
188 }
189
[1e241723]190 if (!found) {
191 /* Find unused PTE in PTEG */
192 for (i = 0; i < 8; i++) {
193 if (!phte[base2 + i].v) {
194 found = true;
195 base = base2;
196 h = 1;
197 break;
198 }
199 }
200 }
201
[7c98874]202 if (!found)
[896ad9f]203 i = RANDI(seed) % 8;
[10e0cee]204 }
205
206 phte[base + i].v = 1;
207 phte[base + i].vsid = vsid;
208 phte[base + i].h = h;
209 phte[base + i].api = api;
[43d6401]210 phte[base + i].rpn = pte->pfn;
[10e0cee]211 phte[base + i].r = 0;
212 phte[base + i].c = 0;
[43d6401]213 phte[base + i].wimg = (pte->page_cache_disable ? WIMG_NO_CACHE : 0);
[10e0cee]214 phte[base + i].pp = 2; // FIXME
215}
216
217
[0867321]218/** Process Instruction/Data Storage Exception
[10e0cee]219 *
[0867321]220 * @param n Exception vector number.
221 * @param istate Interrupted register context.
[10e0cee]222 *
223 */
224void pht_refill(int n, istate_t *istate)
225{
[7f1c620]226 uintptr_t badvaddr;
[10e0cee]227 pte_t *pte;
228 int pfrc;
229 as_t *as;
230 bool lock;
231
232 if (AS == NULL) {
233 as = AS_KERNEL;
234 lock = false;
235 } else {
236 as = AS;
237 lock = true;
238 }
239
[826c203]240 if (n == VECTOR_DATA_STORAGE)
241 badvaddr = istate->dar;
242 else
[10e0cee]243 badvaddr = istate->pc;
244
245 page_table_lock(as, lock);
246
[5d67baa]247 pte = find_mapping_and_check(as, lock, badvaddr,
248 PF_ACCESS_READ /* FIXME */, istate, &pfrc);
[10e0cee]249 if (!pte) {
250 switch (pfrc) {
[5d67baa]251 case AS_PF_FAULT:
252 goto fail;
253 break;
254 case AS_PF_DEFER:
255 /*
256 * The page fault came during copy_from_uspace()
257 * or copy_to_uspace().
258 */
259 page_table_unlock(as, lock);
260 return;
261 default:
[f651e80]262 panic("Unexpected pfrc (%d).", pfrc);
[10e0cee]263 }
264 }
265
[43d6401]266 pte->accessed = 1; /* Record access to PTE */
267 pht_insert(badvaddr, pte);
[10e0cee]268
269 page_table_unlock(as, lock);
270 return;
271
272fail:
273 page_table_unlock(as, lock);
274 pht_refill_fail(badvaddr, istate);
275}
276
277
[0867321]278/** Process Instruction/Data Storage Exception in Real Mode
[10e0cee]279 *
[0867321]280 * @param n Exception vector number.
281 * @param istate Interrupted register context.
[10e0cee]282 *
283 */
[896ad9f]284bool pht_refill_real(int n, istate_t *istate)
[10e0cee]285{
[7f1c620]286 uintptr_t badvaddr;
[10e0cee]287
[826c203]288 if (n == VECTOR_DATA_STORAGE)
289 badvaddr = istate->dar;
290 else
[10e0cee]291 badvaddr = istate->pc;
292
[7f1c620]293 uint32_t physmem;
[10e0cee]294 asm volatile (
295 "mfsprg3 %0\n"
296 : "=r" (physmem)
297 );
298
[896ad9f]299 if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
300 return false;
301
302 uint32_t page = (badvaddr >> 12) & 0xffff;
303 uint32_t api = (badvaddr >> 22) & 0x3f;
304
305 uint32_t vsid;
306 asm volatile (
307 "mfsrin %0, %1\n"
308 : "=r" (vsid)
309 : "r" (badvaddr)
310 );
311
312 uint32_t sdr1;
313 asm volatile (
314 "mfsdr1 %0\n"
315 : "=r" (sdr1)
316 );
317 phte_t *phte_real = (phte_t *) (sdr1 & 0xffff0000);
318
319 /* Primary hash (xor) */
320 uint32_t h = 0;
321 uint32_t hash = vsid ^ page;
322 uint32_t base = (hash & 0x3ff) << 3;
323 uint32_t i;
324 bool found = false;
325
[1e241723]326 /* Find colliding PTE in PTEG */
[896ad9f]327 for (i = 0; i < 8; i++) {
[1e241723]328 if ((phte_real[base + i].v)
329 && (phte_real[base + i].vsid == vsid)
[896ad9f]330 && (phte_real[base + i].api == api)
[1e241723]331 && (phte_real[base + i].h == 0)) {
[896ad9f]332 found = true;
333 break;
334 }
[10e0cee]335 }
336
[1e241723]337 if (!found) {
338 /* Find unused PTE in PTEG */
339 for (i = 0; i < 8; i++) {
340 if (!phte_real[base + i].v) {
341 found = true;
342 break;
343 }
344 }
345 }
346
[896ad9f]347 if (!found) {
348 /* Secondary hash (not) */
349 uint32_t base2 = (~hash & 0x3ff) << 3;
350
[1e241723]351 /* Find colliding PTE in PTEG */
[896ad9f]352 for (i = 0; i < 8; i++) {
[1e241723]353 if ((phte_real[base2 + i].v)
354 && (phte_real[base2 + i].vsid == vsid)
[896ad9f]355 && (phte_real[base2 + i].api == api)
[1e241723]356 && (phte_real[base2 + i].h == 1)) {
[896ad9f]357 found = true;
358 base = base2;
359 h = 1;
360 break;
361 }
362 }
363
[1e241723]364 if (!found) {
365 /* Find unused PTE in PTEG */
366 for (i = 0; i < 8; i++) {
367 if (!phte_real[base2 + i].v) {
368 found = true;
369 base = base2;
370 h = 1;
371 break;
372 }
373 }
374 }
375
[896ad9f]376 if (!found) {
377 /* Use secondary hash to avoid collisions
378 with usual PHT refill handler. */
379 i = RANDI(seed_real) % 8;
380 base = base2;
381 h = 1;
382 }
383 }
384
385 phte_real[base + i].v = 1;
386 phte_real[base + i].vsid = vsid;
387 phte_real[base + i].h = h;
388 phte_real[base + i].api = api;
389 phte_real[base + i].rpn = KA2PA(badvaddr) >> 12;
390 phte_real[base + i].r = 0;
391 phte_real[base + i].c = 0;
392 phte_real[base + i].wimg = 0;
393 phte_real[base + i].pp = 2; // FIXME
394
395 return true;
[10e0cee]396}
397
398
[0867321]399/** Process ITLB/DTLB Miss Exception in Real Mode
400 *
401 *
402 */
403void tlb_refill_real(int n, uint32_t tlbmiss, ptehi_t ptehi, ptelo_t ptelo, istate_t *istate)
404{
405 uint32_t badvaddr = tlbmiss & 0xfffffffc;
406
407 uint32_t physmem;
408 asm volatile (
409 "mfsprg3 %0\n"
410 : "=r" (physmem)
411 );
412
413 if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
414 return; // FIXME
415
416 ptelo.rpn = KA2PA(badvaddr) >> 12;
417 ptelo.wimg = 0;
418 ptelo.pp = 2; // FIXME
419
420 uint32_t index = 0;
421 asm volatile (
422 "mtspr 981, %0\n"
423 "mtspr 982, %1\n"
424 "tlbld %2\n"
425 "tlbli %2\n"
426 : "=r" (index)
427 : "r" (ptehi),
428 "r" (ptelo)
429 );
430}
431
432
[613bc54]433void tlb_arch_init(void)
[a33c990]434{
435 tlb_invalidate_all();
436}
437
438
439void tlb_invalidate_all(void)
[613bc54]440{
[7b187ef]441 uint32_t index;
[9a68b34d]442 asm volatile (
[7b187ef]443 "li %0, 0\n"
444 "sync\n"
445
[e731b0d]446 ".rept 64\n"
447 "tlbie %0\n"
448 "addi %0, %0, 0x1000\n"
449 ".endr\n"
[7b187ef]450
451 "eieio\n"
[a33c990]452 "tlbsync\n"
[7b187ef]453 "sync\n"
454 : "=r" (index)
[9a68b34d]455 );
[613bc54]456}
457
458
[a33c990]459void tlb_invalidate_asid(asid_t asid)
[68965ec5]460{
[7f1c620]461 uint32_t sdr1;
[2c8a70a]462 asm volatile (
463 "mfsdr1 %0\n"
464 : "=r" (sdr1)
465 );
466 phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
467
[7f1c620]468 uint32_t i;
[2c8a70a]469 for (i = 0; i < 8192; i++) {
[5d67baa]470 if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) &&
471 (phte[i].vsid < ((asid << 4) + 16)))
[2c8a70a]472 phte[i].v = 0;
473 }
[a33c990]474 tlb_invalidate_all();
[68965ec5]475}
476
[10e0cee]477
[98000fb]478void tlb_invalidate_pages(asid_t asid, uintptr_t page, size_t cnt)
[a33c990]479{
[10e0cee]480 // TODO
[a33c990]481 tlb_invalidate_all();
482}
483
484
[e600ec4]485#define PRINT_BAT(name, ureg, lreg) \
486 asm volatile ( \
487 "mfspr %0," #ureg "\n" \
488 "mfspr %1," #lreg "\n" \
489 : "=r" (upper), "=r" (lower) \
490 ); \
491 mask = (upper & 0x1ffc) >> 2; \
492 if (upper & 3) { \
[7f1c620]493 uint32_t tmp = mask; \
[e600ec4]494 length = 128; \
495 while (tmp) { \
496 if ((tmp & 1) == 0) { \
497 printf("ibat[0]: error in mask\n"); \
498 break; \
499 } \
500 length <<= 1; \
501 tmp >>= 1; \
502 } \
503 } else \
504 length = 0; \
[5d67baa]505 printf(name ": page=%.*p frame=%.*p length=%d KB (mask=%#x)%s%s\n", \
506 sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, \
507 lower & 0xffff0000, length, mask, \
508 ((upper >> 1) & 1) ? " supervisor" : "", \
509 (upper & 1) ? " user" : "");
[e600ec4]510
511
[613bc54]512void tlb_print(void)
513{
[7f1c620]514 uint32_t sr;
[cf84d72a]515
516 for (sr = 0; sr < 16; sr++) {
[7f1c620]517 uint32_t vsid;
[cf84d72a]518 asm volatile (
519 "mfsrin %0, %1\n"
520 : "=r" (vsid)
521 : "r" (sr << 28)
522 );
[896ad9f]523 printf("sr[%02u]: vsid=%.*p (asid=%u)%s%s\n", sr,
[5d67baa]524 sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4,
525 ((vsid >> 30) & 1) ? " supervisor" : "",
526 ((vsid >> 29) & 1) ? " user" : "");
[cf84d72a]527 }
[e600ec4]528
[7f1c620]529 uint32_t upper;
530 uint32_t lower;
531 uint32_t mask;
532 uint32_t length;
[e600ec4]533
534 PRINT_BAT("ibat[0]", 528, 529);
535 PRINT_BAT("ibat[1]", 530, 531);
536 PRINT_BAT("ibat[2]", 532, 533);
537 PRINT_BAT("ibat[3]", 534, 535);
538
539 PRINT_BAT("dbat[0]", 536, 537);
540 PRINT_BAT("dbat[1]", 538, 539);
541 PRINT_BAT("dbat[2]", 540, 541);
542 PRINT_BAT("dbat[3]", 542, 543);
[613bc54]543}
[b45c443]544
[10e0cee]545/** @}
[b45c443]546 */
Note: See TracBrowser for help on using the repository browser.