source: mainline/kernel/arch/mips32/src/mm/tlb.c@ b4fa652

lfn serial ticket/834-toolchain-update topic/msim-upgrade topic/simplify-dev-export
Last change on this file since b4fa652 was 11675207, checked in by jermar <jermar@…>, 17 years ago

Move everything to kernel/.
  • Property mode set to 100644
File size: 13.0 KB
Line 
1/*
2 * Copyright (C) 2003-2004 Jakub Jermar
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * - Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * - Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * - The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/** @addtogroup mips32mm
30 * @{
31 */
32/** @file
33 */
34
35#include <arch/mm/tlb.h>
36#include <mm/asid.h>
37#include <mm/tlb.h>
38#include <mm/page.h>
39#include <mm/as.h>
40#include <arch/cp0.h>
41#include <panic.h>
42#include <arch.h>
43#include <symtab.h>
44#include <synch/spinlock.h>
45#include <print.h>
46#include <debug.h>
47#include <align.h>
48#include <interrupt.h>
49
50static void tlb_refill_fail(istate_t *istate);
51static void tlb_invalid_fail(istate_t *istate);
52static void tlb_modified_fail(istate_t *istate);
53
54static pte_t *find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate, int *pfrc);
55
56static void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, uintptr_t pfn);
57static void prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr);
58
59/** Initialize TLB
60 *
61 * Initialize TLB.
62 * Invalidate all entries and mark wired entries.
63 */
64void tlb_arch_init(void)
65{
66 int i;
67
68 cp0_pagemask_write(TLB_PAGE_MASK_16K);
69 cp0_entry_hi_write(0);
70 cp0_entry_lo0_write(0);
71 cp0_entry_lo1_write(0);
72
73 /* Clear and initialize TLB. */
74
75 for (i = 0; i < TLB_ENTRY_COUNT; i++) {
76 cp0_index_write(i);
77 tlbwi();
78 }
79
80
81 /*
82 * The kernel is going to make use of some wired
83 * entries (e.g. mapping kernel stacks in kseg3).
84 */
85 cp0_wired_write(TLB_WIRED);
86}
87
88/** Process TLB Refill Exception
89 *
90 * Process TLB Refill Exception.
91 *
92 * @param istate Interrupted register context.
93 */
94void tlb_refill(istate_t *istate)
95{
96 entry_lo_t lo;
97 entry_hi_t hi;
98 asid_t asid;
99 uintptr_t badvaddr;
100 pte_t *pte;
101 int pfrc;
102
103 badvaddr = cp0_badvaddr_read();
104
105 spinlock_lock(&AS->lock);
106 asid = AS->asid;
107 spinlock_unlock(&AS->lock);
108
109 page_table_lock(AS, true);
110
111 pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
112 if (!pte) {
113 switch (pfrc) {
114 case AS_PF_FAULT:
115 goto fail;
116 break;
117 case AS_PF_DEFER:
118 /*
119 * The page fault came during copy_from_uspace()
120 * or copy_to_uspace().
121 */
122 page_table_unlock(AS, true);
123 return;
124 default:
125 panic("unexpected pfrc (%d)\n", pfrc);
126 }
127 }
128
129 /*
130 * Record access to PTE.
131 */
132 pte->a = 1;
133
134 prepare_entry_hi(&hi, asid, badvaddr);
135 prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
136
137 /*
138 * New entry is to be inserted into TLB
139 */
140 cp0_entry_hi_write(hi.value);
141 if ((badvaddr/PAGE_SIZE) % 2 == 0) {
142 cp0_entry_lo0_write(lo.value);
143 cp0_entry_lo1_write(0);
144 }
145 else {
146 cp0_entry_lo0_write(0);
147 cp0_entry_lo1_write(lo.value);
148 }
149 cp0_pagemask_write(TLB_PAGE_MASK_16K);
150 tlbwr();
151
152 page_table_unlock(AS, true);
153 return;
154
155fail:
156 page_table_unlock(AS, true);
157 tlb_refill_fail(istate);
158}
159
160/** Process TLB Invalid Exception
161 *
162 * Process TLB Invalid Exception.
163 *
164 * @param istate Interrupted register context.
165 */
166void tlb_invalid(istate_t *istate)
167{
168 tlb_index_t index;
169 uintptr_t badvaddr;
170 entry_lo_t lo;
171 entry_hi_t hi;
172 pte_t *pte;
173 int pfrc;
174
175 badvaddr = cp0_badvaddr_read();
176
177 /*
178 * Locate the faulting entry in TLB.
179 */
180 hi.value = cp0_entry_hi_read();
181 prepare_entry_hi(&hi, hi.asid, badvaddr);
182 cp0_entry_hi_write(hi.value);
183 tlbp();
184 index.value = cp0_index_read();
185
186 page_table_lock(AS, true);
187
188 /*
189 * Fail if the entry is not in TLB.
190 */
191 if (index.p) {
192 printf("TLB entry not found.\n");
193 goto fail;
194 }
195
196 pte = find_mapping_and_check(badvaddr, PF_ACCESS_READ, istate, &pfrc);
197 if (!pte) {
198 switch (pfrc) {
199 case AS_PF_FAULT:
200 goto fail;
201 break;
202 case AS_PF_DEFER:
203 /*
204 * The page fault came during copy_from_uspace()
205 * or copy_to_uspace().
206 */
207 page_table_unlock(AS, true);
208 return;
209 default:
210 panic("unexpected pfrc (%d)\n", pfrc);
211 }
212 }
213
214 /*
215 * Read the faulting TLB entry.
216 */
217 tlbr();
218
219 /*
220 * Record access to PTE.
221 */
222 pte->a = 1;
223
224 prepare_entry_lo(&lo, pte->g, pte->p, pte->d, pte->cacheable, pte->pfn);
225
226 /*
227 * The entry is to be updated in TLB.
228 */
229 if ((badvaddr/PAGE_SIZE) % 2 == 0)
230 cp0_entry_lo0_write(lo.value);
231 else
232 cp0_entry_lo1_write(lo.value);
233 cp0_pagemask_write(TLB_PAGE_MASK_16K);
234 tlbwi();
235
236 page_table_unlock(AS, true);
237 return;
238
239fail:
240 page_table_unlock(AS, true);
241 tlb_invalid_fail(istate);
242}
243
244/** Process TLB Modified Exception
245 *
246 * Process TLB Modified Exception.
247 *
248 * @param istate Interrupted register context.
249 */
250void tlb_modified(istate_t *istate)
251{
252 tlb_index_t index;
253 uintptr_t badvaddr;
254 entry_lo_t lo;
255 entry_hi_t hi;
256 pte_t *pte;
257 int pfrc;
258
259 badvaddr = cp0_badvaddr_read();
260
261 /*
262 * Locate the faulting entry in TLB.
263 */
264 hi.value = cp0_entry_hi_read();
265 prepare_entry_hi(&hi, hi.asid, badvaddr);
266 cp0_entry_hi_write(hi.value);
267 tlbp();
268 index.value = cp0_index_read();
269
270 page_table_lock(AS, true);
271
272 /*
273 * Fail if the entry is not in TLB.
274 */
275 if (index.p) {
276 printf("TLB entry not found.\n");
277 goto fail;
278 }
279
280 pte = find_mapping_and_check(badvaddr, PF_ACCESS_WRITE, istate, &pfrc);
281 if (!pte) {
282 switch (pfrc) {
283 case AS_PF_FAULT:
284 goto fail;
285 break;
286 case AS_PF_DEFER:
287 /*
288 * The page fault came during copy_from_uspace()
289 * or copy_to_uspace().
290 */
291 page_table_unlock(AS, true);
292 return;
293 default:
294 panic("unexpected pfrc (%d)\n", pfrc);
295 }
296 }
297
298 /*
299 * Fail if the page is not writable.
300 */
301 if (!pte->w)
302 goto fail;
303
304 /*
305 * Read the faulting TLB entry.
306 */
307 tlbr();
308
309 /*
310 * Record access and write to PTE.
311 */
312 pte->a = 1;
313 pte->d = 1;
314
315 prepare_entry_lo(&lo, pte->g, pte->p, pte->w, pte->cacheable, pte->pfn);
316
317 /*
318 * The entry is to be updated in TLB.
319 */
320 if ((badvaddr/PAGE_SIZE) % 2 == 0)
321 cp0_entry_lo0_write(lo.value);
322 else
323 cp0_entry_lo1_write(lo.value);
324 cp0_pagemask_write(TLB_PAGE_MASK_16K);
325 tlbwi();
326
327 page_table_unlock(AS, true);
328 return;
329
330fail:
331 page_table_unlock(AS, true);
332 tlb_modified_fail(istate);
333}
334
335void tlb_refill_fail(istate_t *istate)
336{
337 char *symbol = "";
338 char *sym2 = "";
339
340 char *s = get_symtab_entry(istate->epc);
341 if (s)
342 symbol = s;
343 s = get_symtab_entry(istate->ra);
344 if (s)
345 sym2 = s;
346
347 fault_if_from_uspace(istate, "TLB Refill Exception on %p", cp0_badvaddr_read());
348 panic("%x: TLB Refill Exception at %x(%s<-%s)\n", cp0_badvaddr_read(), istate->epc, symbol, sym2);
349}
350
351
352void tlb_invalid_fail(istate_t *istate)
353{
354 char *symbol = "";
355
356 char *s = get_symtab_entry(istate->epc);
357 if (s)
358 symbol = s;
359 fault_if_from_uspace(istate, "TLB Invalid Exception on %p", cp0_badvaddr_read());
360 panic("%x: TLB Invalid Exception at %x(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
361}
362
363void tlb_modified_fail(istate_t *istate)
364{
365 char *symbol = "";
366
367 char *s = get_symtab_entry(istate->epc);
368 if (s)
369 symbol = s;
370 fault_if_from_uspace(istate, "TLB Modified Exception on %p", cp0_badvaddr_read());
371 panic("%x: TLB Modified Exception at %x(%s)\n", cp0_badvaddr_read(), istate->epc, symbol);
372}
373
374/** Try to find PTE for faulting address
375 *
376 * Try to find PTE for faulting address.
377 * The AS->lock must be held on entry to this function.
378 *
379 * @param badvaddr Faulting virtual address.
380 * @param access Access mode that caused the fault.
381 * @param istate Pointer to interrupted state.
382 * @param pfrc Pointer to variable where as_page_fault() return code will be stored.
383 *
384 * @return PTE on success, NULL otherwise.
385 */
386pte_t *find_mapping_and_check(uintptr_t badvaddr, int access, istate_t *istate, int *pfrc)
387{
388 entry_hi_t hi;
389 pte_t *pte;
390
391 hi.value = cp0_entry_hi_read();
392
393 /*
394 * Handler cannot succeed if the ASIDs don't match.
395 */
396 if (hi.asid != AS->asid) {
397 printf("EntryHi.asid=%d, AS->asid=%d\n", hi.asid, AS->asid);
398 return NULL;
399 }
400
401 /*
402 * Check if the mapping exists in page tables.
403 */
404 pte = page_mapping_find(AS, badvaddr);
405 if (pte && pte->p) {
406 /*
407 * Mapping found in page tables.
408 * Immediately succeed.
409 */
410 return pte;
411 } else {
412 int rc;
413
414 /*
415 * Mapping not found in page tables.
416 * Resort to higher-level page fault handler.
417 */
418 page_table_unlock(AS, true);
419 switch (rc = as_page_fault(badvaddr, access, istate)) {
420 case AS_PF_OK:
421 /*
422 * The higher-level page fault handler succeeded,
423 * The mapping ought to be in place.
424 */
425 page_table_lock(AS, true);
426 pte = page_mapping_find(AS, badvaddr);
427 ASSERT(pte && pte->p);
428 return pte;
429 break;
430 case AS_PF_DEFER:
431 page_table_lock(AS, true);
432 *pfrc = AS_PF_DEFER;
433 return NULL;
434 break;
435 case AS_PF_FAULT:
436 page_table_lock(AS, true);
437 printf("Page fault.\n");
438 *pfrc = AS_PF_FAULT;
439 return NULL;
440 break;
441 default:
442 panic("unexpected rc (%d)\n", rc);
443 }
444
445 }
446}
447
448void prepare_entry_lo(entry_lo_t *lo, bool g, bool v, bool d, bool cacheable, uintptr_t pfn)
449{
450 lo->value = 0;
451 lo->g = g;
452 lo->v = v;
453 lo->d = d;
454 lo->c = cacheable ? PAGE_CACHEABLE_EXC_WRITE : PAGE_UNCACHED;
455 lo->pfn = pfn;
456}
457
458void prepare_entry_hi(entry_hi_t *hi, asid_t asid, uintptr_t addr)
459{
460 hi->value = ALIGN_DOWN(addr, PAGE_SIZE * 2);
461 hi->asid = asid;
462}
463
464/** Print contents of TLB. */
465void tlb_print(void)
466{
467 page_mask_t mask;
468 entry_lo_t lo0, lo1;
469 entry_hi_t hi, hi_save;
470 int i;
471
472 hi_save.value = cp0_entry_hi_read();
473
474 printf("TLB:\n");
475 for (i = 0; i < TLB_ENTRY_COUNT; i++) {
476 cp0_index_write(i);
477 tlbr();
478
479 mask.value = cp0_pagemask_read();
480 hi.value = cp0_entry_hi_read();
481 lo0.value = cp0_entry_lo0_read();
482 lo1.value = cp0_entry_lo1_read();
483
484 printf("%d: asid=%d, vpn2=%d, mask=%d\tg[0]=%d, v[0]=%d, d[0]=%d, c[0]=%hhd, pfn[0]=%d\n"
485 "\t\t\t\tg[1]=%d, v[1]=%d, d[1]=%d, c[1]=%hhd, pfn[1]=%d\n",
486 i, hi.asid, hi.vpn2, mask.mask, lo0.g, lo0.v, lo0.d, lo0.c, lo0.pfn,
487 lo1.g, lo1.v, lo1.d, lo1.c, lo1.pfn);
488 }
489
490 cp0_entry_hi_write(hi_save.value);
491}
492
493/** Invalidate all not wired TLB entries. */
494void tlb_invalidate_all(void)
495{
496 ipl_t ipl;
497 entry_lo_t lo0, lo1;
498 entry_hi_t hi_save;
499 int i;
500
501 hi_save.value = cp0_entry_hi_read();
502 ipl = interrupts_disable();
503
504 for (i = TLB_WIRED; i < TLB_ENTRY_COUNT; i++) {
505 cp0_index_write(i);
506 tlbr();
507
508 lo0.value = cp0_entry_lo0_read();
509 lo1.value = cp0_entry_lo1_read();
510
511 lo0.v = 0;
512 lo1.v = 0;
513
514 cp0_entry_lo0_write(lo0.value);
515 cp0_entry_lo1_write(lo1.value);
516
517 tlbwi();
518 }
519
520 interrupts_restore(ipl);
521 cp0_entry_hi_write(hi_save.value);
522}
523
524/** Invalidate all TLB entries belonging to specified address space.
525 *
526 * @param asid Address space identifier.
527 */
528void tlb_invalidate_asid(asid_t asid)
529{
530 ipl_t ipl;
531 entry_lo_t lo0, lo1;
532 entry_hi_t hi, hi_save;
533 int i;
534
535 ASSERT(asid != ASID_INVALID);
536
537 hi_save.value = cp0_entry_hi_read();
538 ipl = interrupts_disable();
539
540 for (i = 0; i < TLB_ENTRY_COUNT; i++) {
541 cp0_index_write(i);
542 tlbr();
543
544 hi.value = cp0_entry_hi_read();
545
546 if (hi.asid == asid) {
547 lo0.value = cp0_entry_lo0_read();
548 lo1.value = cp0_entry_lo1_read();
549
550 lo0.v = 0;
551 lo1.v = 0;
552
553 cp0_entry_lo0_write(lo0.value);
554 cp0_entry_lo1_write(lo1.value);
555
556 tlbwi();
557 }
558 }
559
560 interrupts_restore(ipl);
561 cp0_entry_hi_write(hi_save.value);
562}
563
564/** Invalidate TLB entries for specified page range belonging to specified address space.
565 *
566 * @param asid Address space identifier.
567 * @param page First page whose TLB entry is to be invalidated.
568 * @param cnt Number of entries to invalidate.
569 */
570void tlb_invalidate_pages(asid_t asid, uintptr_t page, count_t cnt)
571{
572 int i;
573 ipl_t ipl;
574 entry_lo_t lo0, lo1;
575 entry_hi_t hi, hi_save;
576 tlb_index_t index;
577
578 ASSERT(asid != ASID_INVALID);
579
580 hi_save.value = cp0_entry_hi_read();
581 ipl = interrupts_disable();
582
583 for (i = 0; i < cnt+1; i+=2) {
584 hi.value = 0;
585 prepare_entry_hi(&hi, asid, page + i * PAGE_SIZE);
586 cp0_entry_hi_write(hi.value);
587
588 tlbp();
589 index.value = cp0_index_read();
590
591 if (!index.p) {
592 /* Entry was found, index register contains valid index. */
593 tlbr();
594
595 lo0.value = cp0_entry_lo0_read();
596 lo1.value = cp0_entry_lo1_read();
597
598 lo0.v = 0;
599 lo1.v = 0;
600
601 cp0_entry_lo0_write(lo0.value);
602 cp0_entry_lo1_write(lo1.value);
603
604 tlbwi();
605 }
606 }
607
608 interrupts_restore(ipl);
609 cp0_entry_hi_write(hi_save.value);
610}
611
612/** @}
613 */
Note: See TracBrowser for help on using the repository browser.