source: mainline/kernel/arch/ppc32/src/mm/tlb.c@ 40d4c1d

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since 40d4c1d was eae4e8f, checked in by Jakub Jermar <jakub@…>, 16 years ago

Fix missing includes.
  • Property mode set to 100644
File size: 11.5 KB
Line 
1/*
2 * Copyright (c) 2006 Martin Decky
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/** @addtogroup ppc32mm
30 * @{
31 */
32/** @file
33 */
34
35#include <mm/tlb.h>
36#include <arch/mm/tlb.h>
37#include <arch/interrupt.h>
38#include <interrupt.h>
39#include <mm/as.h>
40#include <mm/page.h>
41#include <arch.h>
42#include <print.h>
43#include <macros.h>
44#include <symtab.h>
45
46static unsigned int seed = 10;
47static unsigned int seed_real __attribute__ ((section("K_UNMAPPED_DATA_START"))) = 42;
48
49
50/** Try to find PTE for faulting address
51 *
52 * Try to find PTE for faulting address.
53 * The as->lock must be held on entry to this function
54 * if lock is true.
55 *
56 * @param as Address space.
57 * @param lock Lock/unlock the address space.
58 * @param badvaddr Faulting virtual address.
59 * @param access Access mode that caused the fault.
60 * @param istate Pointer to interrupted state.
61 * @param pfrc Pointer to variable where as_page_fault() return code
62 * will be stored.
63 * @return PTE on success, NULL otherwise.
64 *
65 */
66static pte_t *
67find_mapping_and_check(as_t *as, bool lock, uintptr_t badvaddr, int access,
68 istate_t *istate, int *pfrc)
69{
70 /*
71 * Check if the mapping exists in page tables.
72 */
73 pte_t *pte = page_mapping_find(as, badvaddr);
74 if ((pte) && (pte->present)) {
75 /*
76 * Mapping found in page tables.
77 * Immediately succeed.
78 */
79 return pte;
80 } else {
81 int rc;
82
83 /*
84 * Mapping not found in page tables.
85 * Resort to higher-level page fault handler.
86 */
87 page_table_unlock(as, lock);
88 switch (rc = as_page_fault(badvaddr, access, istate)) {
89 case AS_PF_OK:
90 /*
91 * The higher-level page fault handler succeeded,
92 * The mapping ought to be in place.
93 */
94 page_table_lock(as, lock);
95 pte = page_mapping_find(as, badvaddr);
96 ASSERT((pte) && (pte->present));
97 *pfrc = 0;
98 return pte;
99 case AS_PF_DEFER:
100 page_table_lock(as, lock);
101 *pfrc = rc;
102 return NULL;
103 case AS_PF_FAULT:
104 page_table_lock(as, lock);
105 *pfrc = rc;
106 return NULL;
107 default:
108 panic("Unexpected rc (%d).", rc);
109 }
110 }
111}
112
113
114static void pht_refill_fail(uintptr_t badvaddr, istate_t *istate)
115{
116 char *symbol;
117 char *sym2;
118
119 symbol = symtab_fmt_name_lookup(istate->pc);
120 sym2 = symtab_fmt_name_lookup(istate->lr);
121
122 fault_if_from_uspace(istate,
123 "PHT Refill Exception on %p.", badvaddr);
124 panic("%p: PHT Refill Exception at %p (%s<-%s).", badvaddr,
125 istate->pc, symbol, sym2);
126}
127
128
129static void pht_insert(const uintptr_t vaddr, const pte_t *pte)
130{
131 uint32_t page = (vaddr >> 12) & 0xffff;
132 uint32_t api = (vaddr >> 22) & 0x3f;
133
134 uint32_t vsid;
135 asm volatile (
136 "mfsrin %0, %1\n"
137 : "=r" (vsid)
138 : "r" (vaddr)
139 );
140
141 uint32_t sdr1;
142 asm volatile (
143 "mfsdr1 %0\n"
144 : "=r" (sdr1)
145 );
146 phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
147
148 /* Primary hash (xor) */
149 uint32_t h = 0;
150 uint32_t hash = vsid ^ page;
151 uint32_t base = (hash & 0x3ff) << 3;
152 uint32_t i;
153 bool found = false;
154
155 /* Find colliding PTE in PTEG */
156 for (i = 0; i < 8; i++) {
157 if ((phte[base + i].v)
158 && (phte[base + i].vsid == vsid)
159 && (phte[base + i].api == api)
160 && (phte[base + i].h == 0)) {
161 found = true;
162 break;
163 }
164 }
165
166 if (!found) {
167 /* Find unused PTE in PTEG */
168 for (i = 0; i < 8; i++) {
169 if (!phte[base + i].v) {
170 found = true;
171 break;
172 }
173 }
174 }
175
176 if (!found) {
177 /* Secondary hash (not) */
178 uint32_t base2 = (~hash & 0x3ff) << 3;
179
180 /* Find colliding PTE in PTEG */
181 for (i = 0; i < 8; i++) {
182 if ((phte[base2 + i].v)
183 && (phte[base2 + i].vsid == vsid)
184 && (phte[base2 + i].api == api)
185 && (phte[base2 + i].h == 1)) {
186 found = true;
187 base = base2;
188 h = 1;
189 break;
190 }
191 }
192
193 if (!found) {
194 /* Find unused PTE in PTEG */
195 for (i = 0; i < 8; i++) {
196 if (!phte[base2 + i].v) {
197 found = true;
198 base = base2;
199 h = 1;
200 break;
201 }
202 }
203 }
204
205 if (!found)
206 i = RANDI(seed) % 8;
207 }
208
209 phte[base + i].v = 1;
210 phte[base + i].vsid = vsid;
211 phte[base + i].h = h;
212 phte[base + i].api = api;
213 phte[base + i].rpn = pte->pfn;
214 phte[base + i].r = 0;
215 phte[base + i].c = 0;
216 phte[base + i].wimg = (pte->page_cache_disable ? WIMG_NO_CACHE : 0);
217 phte[base + i].pp = 2; // FIXME
218}
219
220
221/** Process Instruction/Data Storage Exception
222 *
223 * @param n Exception vector number.
224 * @param istate Interrupted register context.
225 *
226 */
227void pht_refill(int n, istate_t *istate)
228{
229 uintptr_t badvaddr;
230 pte_t *pte;
231 int pfrc;
232 as_t *as;
233 bool lock;
234
235 if (AS == NULL) {
236 as = AS_KERNEL;
237 lock = false;
238 } else {
239 as = AS;
240 lock = true;
241 }
242
243 if (n == VECTOR_DATA_STORAGE)
244 badvaddr = istate->dar;
245 else
246 badvaddr = istate->pc;
247
248 page_table_lock(as, lock);
249
250 pte = find_mapping_and_check(as, lock, badvaddr,
251 PF_ACCESS_READ /* FIXME */, istate, &pfrc);
252 if (!pte) {
253 switch (pfrc) {
254 case AS_PF_FAULT:
255 goto fail;
256 break;
257 case AS_PF_DEFER:
258 /*
259 * The page fault came during copy_from_uspace()
260 * or copy_to_uspace().
261 */
262 page_table_unlock(as, lock);
263 return;
264 default:
265 panic("Unexpected pfrc (%d).", pfrc);
266 }
267 }
268
269 pte->accessed = 1; /* Record access to PTE */
270 pht_insert(badvaddr, pte);
271
272 page_table_unlock(as, lock);
273 return;
274
275fail:
276 page_table_unlock(as, lock);
277 pht_refill_fail(badvaddr, istate);
278}
279
280
281/** Process Instruction/Data Storage Exception in Real Mode
282 *
283 * @param n Exception vector number.
284 * @param istate Interrupted register context.
285 *
286 */
287bool pht_refill_real(int n, istate_t *istate)
288{
289 uintptr_t badvaddr;
290
291 if (n == VECTOR_DATA_STORAGE)
292 badvaddr = istate->dar;
293 else
294 badvaddr = istate->pc;
295
296 uint32_t physmem;
297 asm volatile (
298 "mfsprg3 %0\n"
299 : "=r" (physmem)
300 );
301
302 if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
303 return false;
304
305 uint32_t page = (badvaddr >> 12) & 0xffff;
306 uint32_t api = (badvaddr >> 22) & 0x3f;
307
308 uint32_t vsid;
309 asm volatile (
310 "mfsrin %0, %1\n"
311 : "=r" (vsid)
312 : "r" (badvaddr)
313 );
314
315 uint32_t sdr1;
316 asm volatile (
317 "mfsdr1 %0\n"
318 : "=r" (sdr1)
319 );
320 phte_t *phte_real = (phte_t *) (sdr1 & 0xffff0000);
321
322 /* Primary hash (xor) */
323 uint32_t h = 0;
324 uint32_t hash = vsid ^ page;
325 uint32_t base = (hash & 0x3ff) << 3;
326 uint32_t i;
327 bool found = false;
328
329 /* Find colliding PTE in PTEG */
330 for (i = 0; i < 8; i++) {
331 if ((phte_real[base + i].v)
332 && (phte_real[base + i].vsid == vsid)
333 && (phte_real[base + i].api == api)
334 && (phte_real[base + i].h == 0)) {
335 found = true;
336 break;
337 }
338 }
339
340 if (!found) {
341 /* Find unused PTE in PTEG */
342 for (i = 0; i < 8; i++) {
343 if (!phte_real[base + i].v) {
344 found = true;
345 break;
346 }
347 }
348 }
349
350 if (!found) {
351 /* Secondary hash (not) */
352 uint32_t base2 = (~hash & 0x3ff) << 3;
353
354 /* Find colliding PTE in PTEG */
355 for (i = 0; i < 8; i++) {
356 if ((phte_real[base2 + i].v)
357 && (phte_real[base2 + i].vsid == vsid)
358 && (phte_real[base2 + i].api == api)
359 && (phte_real[base2 + i].h == 1)) {
360 found = true;
361 base = base2;
362 h = 1;
363 break;
364 }
365 }
366
367 if (!found) {
368 /* Find unused PTE in PTEG */
369 for (i = 0; i < 8; i++) {
370 if (!phte_real[base2 + i].v) {
371 found = true;
372 base = base2;
373 h = 1;
374 break;
375 }
376 }
377 }
378
379 if (!found) {
380 /* Use secondary hash to avoid collisions
381 with usual PHT refill handler. */
382 i = RANDI(seed_real) % 8;
383 base = base2;
384 h = 1;
385 }
386 }
387
388 phte_real[base + i].v = 1;
389 phte_real[base + i].vsid = vsid;
390 phte_real[base + i].h = h;
391 phte_real[base + i].api = api;
392 phte_real[base + i].rpn = KA2PA(badvaddr) >> 12;
393 phte_real[base + i].r = 0;
394 phte_real[base + i].c = 0;
395 phte_real[base + i].wimg = 0;
396 phte_real[base + i].pp = 2; // FIXME
397
398 return true;
399}
400
401
402/** Process ITLB/DTLB Miss Exception in Real Mode
403 *
404 *
405 */
406void tlb_refill_real(int n, uint32_t tlbmiss, ptehi_t ptehi, ptelo_t ptelo, istate_t *istate)
407{
408 uint32_t badvaddr = tlbmiss & 0xfffffffc;
409
410 uint32_t physmem;
411 asm volatile (
412 "mfsprg3 %0\n"
413 : "=r" (physmem)
414 );
415
416 if ((badvaddr < PA2KA(0)) || (badvaddr >= PA2KA(physmem)))
417 return; // FIXME
418
419 ptelo.rpn = KA2PA(badvaddr) >> 12;
420 ptelo.wimg = 0;
421 ptelo.pp = 2; // FIXME
422
423 uint32_t index = 0;
424 asm volatile (
425 "mtspr 981, %0\n"
426 "mtspr 982, %1\n"
427 "tlbld %2\n"
428 "tlbli %2\n"
429 : "=r" (index)
430 : "r" (ptehi),
431 "r" (ptelo)
432 );
433}
434
435
436void tlb_arch_init(void)
437{
438 tlb_invalidate_all();
439}
440
441
442void tlb_invalidate_all(void)
443{
444 uint32_t index;
445 asm volatile (
446 "li %0, 0\n"
447 "sync\n"
448
449 ".rept 64\n"
450 "tlbie %0\n"
451 "addi %0, %0, 0x1000\n"
452 ".endr\n"
453
454 "eieio\n"
455 "tlbsync\n"
456 "sync\n"
457 : "=r" (index)
458 );
459}
460
461
462void tlb_invalidate_asid(asid_t asid)
463{
464 uint32_t sdr1;
465 asm volatile (
466 "mfsdr1 %0\n"
467 : "=r" (sdr1)
468 );
469 phte_t *phte = (phte_t *) PA2KA(sdr1 & 0xffff0000);
470
471 uint32_t i;
472 for (i = 0; i < 8192; i++) {
473 if ((phte[i].v) && (phte[i].vsid >= (asid << 4)) &&
474 (phte[i].vsid < ((asid << 4) + 16)))
475 phte[i].v = 0;
476 }
477 tlb_invalidate_all();
478}
479
480
481void tlb_invalidate_pages(asid_t asid, uintptr_t page, size_t cnt)
482{
483 // TODO
484 tlb_invalidate_all();
485}
486
487
488#define PRINT_BAT(name, ureg, lreg) \
489 asm volatile ( \
490 "mfspr %0," #ureg "\n" \
491 "mfspr %1," #lreg "\n" \
492 : "=r" (upper), "=r" (lower) \
493 ); \
494 mask = (upper & 0x1ffc) >> 2; \
495 if (upper & 3) { \
496 uint32_t tmp = mask; \
497 length = 128; \
498 while (tmp) { \
499 if ((tmp & 1) == 0) { \
500 printf("ibat[0]: error in mask\n"); \
501 break; \
502 } \
503 length <<= 1; \
504 tmp >>= 1; \
505 } \
506 } else \
507 length = 0; \
508 printf(name ": page=%.*p frame=%.*p length=%d KB (mask=%#x)%s%s\n", \
509 sizeof(upper) * 2, upper & 0xffff0000, sizeof(lower) * 2, \
510 lower & 0xffff0000, length, mask, \
511 ((upper >> 1) & 1) ? " supervisor" : "", \
512 (upper & 1) ? " user" : "");
513
514
515void tlb_print(void)
516{
517 uint32_t sr;
518
519 for (sr = 0; sr < 16; sr++) {
520 uint32_t vsid;
521 asm volatile (
522 "mfsrin %0, %1\n"
523 : "=r" (vsid)
524 : "r" (sr << 28)
525 );
526 printf("sr[%02u]: vsid=%.*p (asid=%u)%s%s\n", sr,
527 sizeof(vsid) * 2, vsid & 0xffffff, (vsid & 0xffffff) >> 4,
528 ((vsid >> 30) & 1) ? " supervisor" : "",
529 ((vsid >> 29) & 1) ? " user" : "");
530 }
531
532 uint32_t upper;
533 uint32_t lower;
534 uint32_t mask;
535 uint32_t length;
536
537 PRINT_BAT("ibat[0]", 528, 529);
538 PRINT_BAT("ibat[1]", 530, 531);
539 PRINT_BAT("ibat[2]", 532, 533);
540 PRINT_BAT("ibat[3]", 534, 535);
541
542 PRINT_BAT("dbat[0]", 536, 537);
543 PRINT_BAT("dbat[1]", 538, 539);
544 PRINT_BAT("dbat[2]", 540, 541);
545 PRINT_BAT("dbat[3]", 542, 543);
546}
547
548/** @}
549 */
Note: See TracBrowser for help on using the repository browser.