source: mainline/kernel/arch/ppc32/src/mm/tlb.c@ 4872160

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since 4872160 was a000878c, checked in by Martin Decky <martin@…>, 15 years ago

make sure that all statically allocated strings are declared as "const char *"
and are treated as read-only
  • Property mode set to 100644
File size: 11.5 KB
Line 
1/*
2 * Copyright (c) 2006 Martin Decky
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/** @addtogroup ppc32mm
30 * @{
31 */
32/** @file
33 */
34
35#include <mm/tlb.h>
36#include <arch/mm/tlb.h>
37#include <arch/interrupt.h>
38#include <interrupt.h>
39#include <mm/as.h>
40#include <mm/page.h>
41#include <arch.h>
42#include <print.h>
43#include <macros.h>
44#include <symtab.h>
45
46static unsigned int seed = 10;
47static unsigned int seed_real __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42;
48
49
50/** Try to find PTE for faulting address
51 *
52 * Try to find PTE for faulting address.
53 * The as->lock must be held on entry to this function
54 * if lock is true.
55 *
56 * @param as Address space.
57 * @param lock Lock/unlock the address space.
58 * @param badvaddr Faulting virtual address.
59 * @param access Access mode that caused the fault.
60 * @param istate Pointer to interrupted state.
61 * @param pfrc Pointer to variable where as_page_fault() return code
62 * will be stored.
63 * @return PTE on success, NULL otherwise.
64 *
65 */
66static pte_t *
67find_mapping_and_check(as_t *as, bool lock, uintptr_t badvaddr, int access,
68 istate_t *istate, int *pfrc)
69{
70 /*
71 * Check if the mapping exists in page tables.
72 */
73 pte_t *pte = page_mapping_find(as, badvaddr);
74 if ((pte) && (pte->present)) {
75 /*
76 * Mapping found in page tables.
77 * Immediately succeed.
78 */
79 return pte;
80 } else {
81 int rc;
82
83 /*
84 * Mapping not found in page tables.
85 * Resort to higher-level page fault handler.
86 */
87 page_table_unlock(as, lock);
88 switch (rc = as_page_fault(badvaddr, access, istate)) {
89 case AS_PF_OK:
90 /*
91 * The higher-level page fault handler succeeded,
92 * The mapping ought to be in place.
93 */
94 page_table_lock(as, lock);
95 pte = page_mapping_find(as, badvaddr);
96 ASSERT((pte) && (pte->present));
97 *pfrc = 0;
98 return pte;
99 case AS_PF_DEFER:
100 page_table_lock(as, lock);
101 *pfrc = rc;
102 return NULL;
103 case AS_PF_FAULT:
104 page_table_lock(as, lock);
105 *pfrc = rc;
106 return NULL;
107 default:
108 panic("Unexpected rc (%d).", rc);
109 }
110 }
111}
112
113
114static void pht_refill_fail(uintptr_t badvaddr, istate_t *istate)
115{
116 const char *symbol = symtab_fmt_name_lookup(istate->pc);
117 const char *sym2 = symtab_fmt_name_lookup(istate->lr);
118
119 fault_if_from_uspace(istate,
120 "PHT Refill Exception on %p.", badvaddr);
121 panic("%p: PHT Refill Exception at %p (%s<-%s).", badvaddr,
122 istate->pc, symbol, sym2);
123}
124
125
126static void pht_insert(const uintptr_t vaddr, const pte_t *pte)
127{
128 uint32_t page = (vaddr >> 12) & 0xffff;
129 uint32_t api = (vaddr >> 22) & 0x3f;
130
131 uint32_t vsid;
132 asm volatile (
133 "mfsrin %0, %1\n"
134 : "=r" (vsid)
135 : "r" (vaddr)
136 );
137
138 uint32_t sdr1;
139 asm volatile (
140 "mfsdr1 %0\n"
141 : "=r" (sdr1)
142 );
143 phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
144
145 /* Primary hash (xor) */
146 uint32_t h = 0;
147 uint32_t hash = vsid ^ page;
148 uint32_t base = (hash & 0x3ff) << 3;
149 uint32_t i;
150 bool found = false;
151
152 /* Find colliding PTE in PTEG */
153 for (i = 0; i < 8; i++) {
154 if ((phte[base + i].v)
155 && (phte[base + i].vsid == vsid)
156 && (phte[base + i].api == api)
157 && (phte[base + i].h == 0)) {
158 found = true;
159 break;
160 }
161 }
162
163 if (!found) {
164 /* Find unused PTE in PTEG */
165 for (i = 0; i < 8; i++) {
166 if (!phte[base + i].v) {
167 found = true;
168 break;
169 }
170 }
171 }
172
173 if (!found) {
174 /* Secondary hash (not) */
175 uint32_t base2 = (~hash & 0x3ff) << 3;
176
177 /* Find colliding PTE in PTEG */
178 for (i = 0; i < 8; i++) {
179 if ((phte[base2 + i].v)
180 && (phte[base2 + i].vsid == vsid)
181 && (phte[base2 + i].api == api)
182 && (phte[base2 + i].h == 1)) {
183 found = true;
184 base = base2;
185 h = 1;
186 break;
187 }
188 }
189
190 if (!found) {
191 /* Find unused PTE in PTEG */
192 for (i = 0; i < 8; i++) {
193 if (!phte[base2 + i].v) {
194 found = true;
195 base = base2;
196 h = 1;
197 break;
198 }
199 }
200 }
201
202 if (!found)
203 i = RANDI(seed) % 8;
204 }
205
206 phte[base + i].v = 1;
207 phte[base + i].vsid = vsid;
208 phte[base + i].h = h;
209 phte[base + i].api = api;
210 phte[base + i].rpn = pte->pfn;
211 phte[base + i].r = 0;
212 phte[base + i].c = 0;
213 phte[base + i].wimg = (pte->page_cache_disable ? WIMG_NO_CACHE : 0);
214 phte[base + i].pp = 2; // FIXME
215}
216
217
218/** Process Instruction/Data Storage Exception
219 *
220 * @param n Exception vector number.
221 * @param istate Interrupted register context.
222 *
223 */
224void pht_refill(int n, istate_t *istate)
225{
226 uintptr_t badvaddr;
227 pte_t *pte;
228 int pfrc;
229 as_t *as;
230 bool lock;
231
232 if (AS == NULL) {
233 as = AS_KERNEL;
234 lock = false;
235 } else {
236 as = AS;
237 lock = true;
238 }
239
240 if (n == VECTOR_DATA_STORAGE)
241 badvaddr = istate->dar;
242 else
243 badvaddr = istate->pc;
244
245 page_table_lock(as, lock);
246
247 pte = find_mapping_and_check(as, lock, badvaddr,
248 PF_ACCESS_READ /* FIXME */, istate, &pfrc);
249 if (!pte) {
250 switch (pfrc) {
251 case AS_PF_FAULT:
252 goto fail;
253 break;
254 case AS_PF_DEFER:
255 /*
256 * The page fault came during copy_from_uspace()
257 * or copy_to_uspace().
258 */
259 page_table_unlock(as, lock);
260 return;
261 default:
262 panic("Unexpected pfrc (%d).", pfrc);
263 }
264 }
265
266 pte->accessed = 1; /* Record access to PTE */
267 pht_insert(badvaddr, pte);
268
269 page_table_unlock(as, lock);
270 return;
271
272fail:
273 page_table_unlock(as, lock);
274 pht_refill_fail(badvaddr, istate);
275}
276
277
278/** Process Instruction/Data Storage Exception in Real Mode
279 *
280 * @param n Exception vector number.
281 * @param istate Interrupted register context.
282 *
283 */
284bool pht_refill_real(int n, istate_t *istate)
285{
286 uintptr_t badvaddr;
287
288 if (n == VECTOR_DATA_STORAGE)
289 badvaddr = istate->dar;
290 else
291 badvaddr = istate->pc;
292
293 uint32_t physmem;
294 asm volatile (
295 "mfsprg3 %0\n"
296 : "=r" (physmem)
297 );
298
299 if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
300 return false;
301
302 uint32_t page = (badvaddr >> 12) & 0xffff;
303 uint32_t api = (badvaddr >> 22) & 0x3f;
304
305 uint32_t vsid;
306 asm volatile (
307 "mfsrin %0, %1\n"
308 : "=r" (vsid)
309 : "r" (badvaddr)
310 );
311
312 uint32_t sdr1;
313 asm volatile (
314 "mfsdr1 %0\n"
315 : "=r" (sdr1)
316 );
317 phte_t *phte_real = (phte_t *) (sdr1 & 0xffff0000);
318
319 /* Primary hash (xor) */
320 uint32_t h = 0;
321 uint32_t hash = vsid ^ page;
322 uint32_t base = (hash & 0x3ff) << 3;
323 uint32_t i;
324 bool found = false;
325
326 /* Find colliding PTE in PTEG */
327 for (i = 0; i < 8; i++) {
328 if ((phte_real[base + i].v)
329 && (phte_real[base + i].vsid == vsid)
330 && (phte_real[base + i].api == api)
331 && (phte_real[base + i].h == 0)) {
332 found = true;
333 break;
334 }
335 }
336
337 if (!found) {
338 /* Find unused PTE in PTEG */
339 for (i = 0; i < 8; i++) {
340 if (!phte_real[base + i].v) {
341 found = true;
342 break;
343 }
344 }
345 }
346
347 if (!found) {
348 /* Secondary hash (not) */
349 uint32_t base2 = (~hash & 0x3ff) << 3;
350
351 /* Find colliding PTE in PTEG */
352 for (i = 0; i < 8; i++) {
353 if ((phte_real[base2 + i].v)
354 && (phte_real[base2 + i].vsid == vsid)
355 && (phte_real[base2 + i].api == api)
356 && (phte_real[base2 + i].h == 1)) {
357 found = true;
358 base = base2;
359 h = 1;
360 break;
361 }
362 }
363
364 if (!found) {
365 /* Find unused PTE in PTEG */
366 for (i = 0; i < 8; i++) {
367 if (!phte_real[base2 + i].v) {
368 found = true;
369 base = base2;
370 h = 1;
371 break;
372 }
373 }
374 }
375
376 if (!found) {
377 /* Use secondary hash to avoid collisions
378 with usual PHT refill handler. */
379 i = RANDI(seed_real) % 8;
380 base = base2;
381 h = 1;
382 }
383 }
384
385 phte_real[base + i].v = 1;
386 phte_real[base + i].vsid = vsid;
387 phte_real[base + i].h = h;
388 phte_real[base + i].api = api;
389 phte_real[base + i].rpn = KA2PA(badvaddr) >> 12;
390 phte_real[base + i].r = 0;
391 phte_real[base + i].c = 0;
392 phte_real[base + i].wimg = 0;
393 phte_real[base + i].pp = 2; // FIXME
394
395 return true;
396}
397
398
399/** Process ITLB/DTLB Miss Exception in Real Mode
400 *
401 *
402 */
403void tlb_refill_real(int n, uint32_t tlbmiss, ptehi_t ptehi, ptelo_t ptelo, istate_t *istate)
404{
405 uint32_t badvaddr = tlbmiss & 0xfffffffc;
406
407 uint32_t physmem;
408 asm volatile (
409 "mfsprg3 %0\n"
410 : "=r" (physmem)
411 );
412
413 if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
414 return; // FIXME
415
416 ptelo.rpn = KA2PA(badvaddr) >> 12;
417 ptelo.wimg = 0;
418 ptelo.pp = 2; // FIXME
419
420 uint32_t index = 0;
421 asm volatile (
422 "mtspr 981, %0\n"
423 "mtspr 982, %1\n"
424 "tlbld %2\n"
425 "tlbli %2\n"
426 : "=r" (index)
427 : "r" (ptehi),
428 "r" (ptelo)
429 );
430}
431
432
433void tlb_arch_init(void)
434{
435 tlb_invalidate_all();
436}
437
438
439void tlb_invalidate_all(void)
440{
441 uint32_t index;
442 asm volatile (
443 "li %0, 0\n"
444 "sync\n"
445
446 ".rept 64\n"
447 "tlbie %0\n"
448 "addi %0, %0, 0x1000\n"
449 ".endr\n"
450
451 "eieio\n"
452 "tlbsync\n"
453 "sync\n"
454 : "=r" (index)
455 );
456}
457
458
459void tlb_invalidate_asid(asid_t asid)
460{
461 uint32_t sdr1;
462 asm volatile (
463 "mfsdr1 %0\n"
464 : "=r" (sdr1)
465 );
466 phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
467
468 uint32_t i;
469 for (i = 0; i < 8192; i++) {
470 if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) &&
471 (phte[i].vsid < ((asid << 4) + 16)))
472 phte[i].v = 0;
473 }
474 tlb_invalidate_all();
475}
476
477
478void tlb_invalidate_pages(asid_t asid, uintptr_t page, size_t cnt)
479{
480 // TODO
481 tlb_invalidate_all();
482}
483
484
485#define PRINT_BAT(name, ureg, lreg) \
486 asm volatile ( \
487 "mfspr %0," #ureg "\n" \
488 "mfspr %1," #lreg "\n" \
489 : "=r" (upper), "=r" (lower) \
490 ); \
491 mask = (upper & 0x1ffc) >> 2; \
492 if (upper & 3) { \
493 uint32_t tmp = mask; \
494 length = 128; \
495 while (tmp) { \
496 if ((tmp & 1) == 0) { \
497 printf("ibat[0]: error in mask\n"); \
498 break; \
499 } \
500 length <<= 1; \
501 tmp >>= 1; \
502 } \
503 } else \
504 length = 0; \
505 printf(name ": page=%.*p frame=%.*p length=%d KB (mask=%#x)%s%s\n", \
506 sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, \
507 lower & 0xffff0000, length, mask, \
508 ((upper >> 1) & 1) ? " supervisor" : "", \
509 (upper & 1) ? " user" : "");
510
511
512void tlb_print(void)
513{
514 uint32_t sr;
515
516 for (sr = 0; sr < 16; sr++) {
517 uint32_t vsid;
518 asm volatile (
519 "mfsrin %0, %1\n"
520 : "=r" (vsid)
521 : "r" (sr << 28)
522 );
523 printf("sr[%02u]: vsid=%.*p (asid=%u)%s%s\n", sr,
524 sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4,
525 ((vsid >> 30) & 1) ? " supervisor" : "",
526 ((vsid >> 29) & 1) ? " user" : "");
527 }
528
529 uint32_t upper;
530 uint32_t lower;
531 uint32_t mask;
532 uint32_t length;
533
534 PRINT_BAT("ibat[0]", 528, 529);
535 PRINT_BAT("ibat[1]", 530, 531);
536 PRINT_BAT("ibat[2]", 532, 533);
537 PRINT_BAT("ibat[3]", 534, 535);
538
539 PRINT_BAT("dbat[0]", 536, 537);
540 PRINT_BAT("dbat[1]", 538, 539);
541 PRINT_BAT("dbat[2]", 540, 541);
542 PRINT_BAT("dbat[3]", 542, 543);
543}
544
545/** @}
546 */
Note: See TracBrowser for help on using the repository browser.